-
传统Seq2Seq模型的缺陷
1）encoder 最后一个 hidden state，与句子末端词汇的关联较大，难以保留句子起始部分的信息。因此当句子过长时，模型性能下降很快。2）句子中每个词都赋予相同的权重的做法是不合理的，这样没有足够的区分度。
-
Attention Based Model 则打破传统 Seq2Seq 模型在编码解码的时候都依赖于内部一个固定长度的向量的限制。
Seq2Seq vs. Attention based Model Attention Based Model 优势在于不管序列多长，中间训练出来的 context 能够保留序列上的每个输入点的信息以及序列每个输入点之间的关联
-
Attention Based Model 架构
- encoder：(multi-rnn) LSTM / GRU
- attention：softmax + attention metric
- decoder：(multi-rnn) LSTM / GRU
Attention Model
Attention Metric
相对于原本的Seq2Seq模型，
加入Attention机制的模型，
相当于每个输出位置有一个对应的向量。其中，
表示输入经过encoder层嵌入后得到的结果
Attention Model 实现
self.encoder_inputs = tf.placeholder(tf.int32, shape=[None, None], name='encoder_inputs')
self.decoder_inputs = tf.placeholder(tf.int32, shape=[None, None], name='decoder_inputs')
self.decoder_targets = tf.placeholder(tf.int32, shape=[None, None], name='decoder_targets')
self.encoder_length = tf.placeholder(tf.int32, shape=[None], name='encoder_length')
self.decoder_length = tf.placeholder(tf.int32, shape=[None], name='decoder_length')
with tf.variable_scope("embedding"):
encoder_embedding = tf.Variable(tf.truncated_normal(shape=[self.options["vocab_size"], self.options["embedding_dim"]], name='encoder_embedding'))
decoder_embedding = tf.Variable(tf.truncated_normal(shape=[self.options["vocab_size"], self.options["embedding_dim"]], name='decoder_embedding'))
# embedding
self.encoder_inputs_embedded = tf.nn.embedding_lookup(encoder_embedding, self.encoder_inputs)
self.decoder_inputs_embedded = tf.nn.embedding_lookup(decoder_embedding, self.decoder_inputs)
with tf.name_scope("encoder"):
# encoder
encoder_layers = [tf.nn.rnn_cell.BasicLSTMCell(self.options["cell_size"]) for _ in range(2)]
encoder = tf.nn.rnn_cell.MultiRNNCell(encoder_layers)
# encoder output
encoder_all_outputs, encoder_final_state = tf.nn.dynamic_rnn(encoder, self.encoder_inputs_embedded, sequence_length=self.encoder_length, dtype=tf.float32, time_major=False)
if self.is_summaries:
tf.summary.histogram('encoder_all_outputs', encoder_all_outputs)
with tf.name_scope("decoder"):
# decoder init
decoder_layers = [tf.nn.rnn_cell.BasicLSTMCell(self.options["cell_size"]) for _ in range(2)]
decoder_cell = tf.nn.rnn_cell.MultiRNNCell(decoder_layers)
# attention
attention_mechanism = LuongAttention(num_units=self.options["cell_size"], memory=encoder_all_outputs, memory_sequence_length=self.encoder_length)
# decoder & attention concat
attention_decoder = AttentionWrapper(cell=decoder_cell, attention_mechanism=attention_mechanism, alignment_history=True, output_attention=True)
attention_initial_state = attention_decoder.zero_state(tf.shape(self.encoder_inputs)[0], tf.float32).clone(cell_state=encoder_final_state)
# assign decoder input
train_helper = TrainingHelper(self.decoder_inputs_embedded, self.decoder_length)
# define decoder output_layer
fc_layer = tf.layers.Dense(self.options["vocab_size"])
train_decoder = BasicDecoder(cell=attention_decoder, helper=train_helper, initial_state=attention_initial_state, output_layer=fc_layer)
# decoder output
logits, final_state, final_sequence_lengths = dynamic_decode(train_decoder)
decoder_logits = logits.rnn_output
self.train_attention_matrices = final_state.alignment_history.stack(name="train_attention_matrix")
if self.is_summaries:
tf.summary.histogram('decoder_logits', decoder_logits)
tf.summary.histogram('train_attention_matrix', self.train_attention_matrices)
with tf.name_scope("train"):
maxlen = tf.reduce_max(self.decoder_length, name="mask_max_len")
mask = tf.sequence_mask(self.decoder_length, maxlen=maxlen, dtype=tf.float32, name="mask")
decoder_labels = tf.one_hot(self.decoder_targets, depth=self.options["vocab_size"], dtype=tf.int32, name="decoder_labels")
stepwise_cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(labels=decoder_labels, logits=decoder_logits, name="cross_entropy")
_loss = tf.multiply(stepwise_cross_entropy, mask)
self.loss = tf.reduce_sum(_loss, name="loss")
optimizer = tf.train.AdadeltaOptimizer()
self.train_op = optimizer.minimize(self.loss)
if self.is_summaries:
tf.summary.histogram('loss', self.loss)
