We can inherit tensorflow RNNCell to create a custom RNN and create some variables in build() function. However, we may get this error: ValueError: Initializer for variable is from inside a control-flow construct, such as a loop or conditional. When creating a variable inside a loop or conditional, use a lambda as the initializer.
In this tutorial, we will introduce you how to fix it.
For example:
class CustomCell(tf.nn.rnn_cell.RNNCell): def __init__(self, num_units, reuse = None, dtype = tf.float32, name = "custom_cell"): super(CustomCell, self).__init__(_reuse=reuse, dtype = dtype, name=name) self._num_units = num_units # the dimension of rnn cell @property def state_size(self): return self._num_units @property def output_size(self): return self._num_units def build(self, inputs_shape): inputs_dim = inputs_shape[-1].value self._w = tf.Variable(tf.truncated_normal([inputs_dim, self._num_units], stddev=0.1), name="weight") self._b = tf.Variable(tf.truncated_normal([self._num_units], stddev=0.1), name="bias") self.built = True def call(self, inputs, state): # call body # how to use previous rnn cell output and state to generate new output and hidden new_h = tf.tanh(tf.matmul(inputs+state, self._w) + self._b) new_c = tf.nn.swish(tf.matmul(state, self._w) + self._b) return new_h, new_c
If you use this CustomCell class, you will get this error:
How to fix this ValueError?
We can not create tensorflow variables in build() function using tf.Variable(). We should use self.add_variable().
For example:
class CustomCell(tf.nn.rnn_cell.RNNCell): def __init__(self, num_units, reuse = None, dtype = tf.float32, name = "custom_cell"): super(CustomCell, self).__init__(_reuse=reuse, dtype = dtype, name=name) self._num_units = num_units # the dimension of rnn cell @property def state_size(self): return self._num_units @property def output_size(self): return self._num_units def build(self, inputs_shape): inputs_dim = inputs_shape[-1].value self._w = self.add_variable(name="weight", shape = [inputs_dim, self._num_units], initializer = tf.glorot_normal_initializer(), dtype = tf.float32) self._b = self.add_variable(name="bias", shape=[self._num_units], initializer=tf.glorot_normal_initializer(), dtype=tf.float32) self.built = True def call(self, inputs, state): # call body # how to use previous rnn cell output and state to generate new output and hidden new_h = tf.tanh(tf.matmul(inputs+state, self._w) + self._b) new_c = tf.nn.swish(tf.matmul(state, self._w) + self._b) return new_h, new_c
Then we can use code below to evaluate it.
size = 100
inputs = tf.Variable(tf.truncated_normal([3, 20, 100], stddev=0.1), name="inputs")
input_lengths = tf.Variable(tf.truncated_normal([3, 20], stddev=0.1), name="inputs_length")
_fw_cell =CustomCell(size, name='encoder_fw_')
_bw_cell =CustomCell(size, name='encoder_bw')
with tf.variable_scope("Custom_BiLSTM"):
outputs, (fw_state, bw_state) = tf.nn.bidirectional_dynamic_rnn(
_fw_cell,
_bw_cell,
inputs,
sequence_length=None,
dtype=tf.float32,
swap_memory=True)
outputs = tf.concat(outputs, axis=2) # Concat and return forward + backward outputs
init = tf.global_variables_initializer()
init_local = tf.local_variables_initializer()
with tf.Session() as sess:
sess.run([init, init_local])
np.set_printoptions(precision=4, suppress=True)
f =sess.run([inputs, outputs])
Run this code, we will get:
f shape= (3, 20, 100) (3, 20, 200)
[array([[[ 0.1409, 0.107 , 0.0258, ..., 0.0281, -0.0612, 0.0525],
[ 0.0652, -0.0202, 0.0169, ..., -0.1956, 0.0543, -0.0334],
[-0.0559, 0.1613, 0.0257, ..., 0.0858, 0.1105, -0.0963],
...,
[-0.0716, -0.0563, -0.0451, ..., 0.1238, -0.0111, -0.0465],
[-0.0484, 0.0344, -0.0566, ..., -0.1707, -0.0705, 0.01 ],
[-0.0037, -0.0209, -0.0565, ..., 0.0233, 0.0548, 0.1174]]],
dtype=float32), array([[[-0.0311, -0.246 , -0.0412, ..., -0.0077, -0.0249, 0.0986],
[-0.1012, -0.1396, -0.0219, ..., -0.0893, 0.128 , 0.1197],
[ 0.0678, -0.2054, -0.0608, ..., -0.0068, -0.0627, 0.1589],
...,
[ 0.0128, -0.1721, 0.0594, ..., 0.1566, 0.1048, -0.104 ],
[-0.0187, -0.2965, 0.0667, ..., -0.0013, 0.0132, 0.046 ],
[ 0.0082, -0.1131, 0.0417, ..., 0.068 , 0.2191, 0.0546]]],
dtype=float32)]
This value error is fixed.