Tensorflow:使用raw_rnn复制dynamic_rnn行为

Question

我正在尝试使用低级api tf.nn.raw_rnn复制tf.nn.dynamic_rnn的行为。为了做到这一点，我使用相同的数据补丁，设置随机种子，并使用相同的hparams来创建细胞和递归神经网络。但是，这两种实现生成的输出并不相等。下面是数据和代码。

data和lengths

X = np.array([[[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [0.0, 0.0, 0.0]], [[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [7.7, 8.8, 9.9]], [[1.1, 2.2, 3.3], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], dtype=np.float32)
X_len = np.array([2, 3, 1], dtype=np.int32)

tf.nn.dynamic_rnn实现：

tf.reset_default_graph()
tf.set_random_seed(42)

inputs = tf.placeholder(shape=(3, None, 3), dtype=tf.float32)
lengths = tf.placeholder(shape=(None,), dtype=tf.int32)

lstm_cell = tf.nn.rnn_cell.LSTMCell(5)
outputs, state = tf.nn.dynamic_rnn(inputs=inputs, sequence_length=lengths, cell=lstm_cell, dtype=tf.float32, initial_state=lstm_cell.zero_state(3, dtype=tf.float32), time_major=True)
outputs_reshaped = tf.transpose(outputs, perm=[1, 0, 2])

sess = tf.Session()
sess.run(tf.initializers.global_variables())
X = np.transpose(X, (1, 0, 2))
hidden_state = sess.run(outputs_reshaped, feed_dict={inputs: X, lengths: X_len})
print(hidden_state)

tf.nn.raw_rnn实现：

tf.reset_default_graph()
tf.set_random_seed(42)

inputs = tf.placeholder(shape=(3, None, 3),dtype=tf.float32)
lengths = tf.placeholder(shape=(None,), dtype=tf.int32)

inputs_ta = tf.TensorArray(dtype=tf.float32, size=3)
inputs_ta = inputs_ta.unstack(inputs)

lstm_cell = tf.nn.rnn_cell.LSTMCell(5)

def loop_fn(time, cell_output, cell_state, loop_state):
    emit_output = cell_output  # == None for time == 0
    if cell_output is None:  # time == 0
        next_cell_state = lstm_cell.zero_state(3, tf.float32)
    else:
        next_cell_state = cell_state

    elements_finished = (time >= lengths)
    finished = tf.reduce_all(elements_finished)
    next_input = tf.cond(finished, true_fn=lambda: tf.zeros([3, 3], dtype=tf.float32), false_fn=lambda: inputs_ta.read(time))

    next_loop_state = None

    return (elements_finished, next_input, next_cell_state, emit_output, next_loop_state)

outputs_ta, final_state, _ = tf.nn.raw_rnn(lstm_cell, loop_fn)
outputs_reshaped = tf.transpose(outputs_ta.stack(), perm=[1, 0, 2])

sess = tf.Session()
sess.run(tf.initializers.global_variables())

X = np.transpose(X, (1, 0, 2))
hidden_state = sess.run(outputs_reshaped, feed_dict={inputs: X, lengths: X_len})

print(hidden_state)

我确信它们之间存在一些差异，但我无法弄清楚它在哪里和什么地方。如果任何人有一个想法，那将是非常棒的。

期待您的回答！

Answer 1

出现差异的原因是您的变量初始化为不同的值。您可以通过调用以下命令来查看：

print(sess.run(tf.trainable_variables()))

在初始化它们之后。

出现这种差异的原因是有一个全局种子和一个每个操作的种子，因此设置随机种子不会强制调用隐藏在lstm代码中的初始化程序使用相同的随机种子。请参阅this answer for more details on this。总而言之:用于任何随机的随机种子，从你的全局种子开始，然后取决于添加到图中的最后一个操作的id。

了解了这一点，我们可以通过以完全相同的顺序构建图直到构造变量来强制两个实现中的变量种子相同:这意味着我们从相同的全局种子开始，并以相同的顺序将相同的操作添加到图中，直到变量，因此变量将具有相同的操作种子。我们可以这样做：

tf.reset_default_graph()
tf.set_random_seed(42)
lstm_cell = tf.nn.rnn_cell.LSTMCell(5)
inputs_shape = (3, None, 3)
lstm_cell.build(inputs_shape)

build方法是必需的，因为这实际上是将变量添加到图中的方法。

以下是您所拥有的完整工作版本：

import tensorflow as tf
import numpy as np


X = np.array([[[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [0.0, 0.0, 0.0]], [[1.1, 2.2, 3.3], [4.4, 5.5, 6.6], [7.7, 8.8, 9.9]], [[1.1, 2.2, 3.3], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]], dtype=np.float32)
X_len = np.array([2, 3, 1], dtype=np.int32)


def dynamic():
    tf.reset_default_graph()
    tf.set_random_seed(42)
    lstm_cell = tf.nn.rnn_cell.LSTMCell(5)
    inputs_shape = (3, None, 3)
    lstm_cell.build(inputs_shape)

    inputs = tf.placeholder(shape=inputs_shape, dtype=tf.float32)
    lengths = tf.placeholder(shape=(None,), dtype=tf.int32)

    outputs, state = tf.nn.dynamic_rnn(inputs=inputs, sequence_length=lengths, cell=lstm_cell, dtype=tf.float32,
                                       initial_state=lstm_cell.zero_state(3, dtype=tf.float32), time_major=True)
    outputs_reshaped = tf.transpose(outputs, perm=[1, 0, 2])

    sess = tf.Session()
    sess.run(tf.initializers.global_variables())
    a = np.transpose(X, (1, 0, 2))
    hidden_state = sess.run(outputs_reshaped, feed_dict={inputs: a, lengths: X_len})
    print(hidden_state)


def replicated():
    tf.reset_default_graph()
    tf.set_random_seed(42)
    lstm_cell = tf.nn.rnn_cell.LSTMCell(5)
    inputs_shape = (3, None, 3)
    lstm_cell.build(inputs_shape)

    inputs = tf.placeholder(shape=inputs_shape, dtype=tf.float32)
    lengths = tf.placeholder(shape=(None,), dtype=tf.int32)

    inputs_ta = tf.TensorArray(dtype=tf.float32, size=3)
    inputs_ta = inputs_ta.unstack(inputs)


    def loop_fn(time, cell_output, cell_state, loop_state):
        emit_output = cell_output  # == None for time == 0
        if cell_output is None:  # time == 0
            next_cell_state = lstm_cell.zero_state(3, tf.float32)
        else:
            next_cell_state = cell_state

        elements_finished = (time >= lengths)
        finished = tf.reduce_all(elements_finished)
        next_input = tf.cond(finished, true_fn=lambda: tf.zeros([3, 3], dtype=tf.float32),
                             false_fn=lambda: inputs_ta.read(time))

        next_loop_state = None

        return (elements_finished, next_input, next_cell_state, emit_output, next_loop_state)

    outputs_ta, final_state, _ = tf.nn.raw_rnn(lstm_cell, loop_fn)
    outputs_reshaped = tf.transpose(outputs_ta.stack(), perm=[1, 0, 2])

    sess = tf.Session()
    sess.run(tf.initializers.global_variables())

    a = np.transpose(X, (1, 0, 2))
    hidden_state = sess.run(outputs_reshaped, feed_dict={inputs: a, lengths: X_len})

    print(hidden_state)


if __name__ == '__main__':
    dynamic()
    replicated()