From 4c4de46cf21d98daeebb897b11efba380d294a0f Mon Sep 17 00:00:00 2001 From: Yutaka Leon Date: Thu, 7 Jan 2016 13:26:35 -0800 Subject: [PATCH] Update init_ops to also support initialization of float64 variables. Change: 111625948 --- tensorflow/g3doc/api_docs/python/state_ops.md | 40 +++++++-- .../python/kernel_tests/init_ops_test.py | 88 ++++++++++++------- tensorflow/python/ops/init_ops.py | 68 +++++++++++--- 3 files changed, 150 insertions(+), 46 deletions(-) diff --git a/tensorflow/g3doc/api_docs/python/state_ops.md b/tensorflow/g3doc/api_docs/python/state_ops.md index 7a183236855..504ecc7e243 100644 --- a/tensorflow/g3doc/api_docs/python/state_ops.md +++ b/tensorflow/g3doc/api_docs/python/state_ops.md @@ -1164,7 +1164,7 @@ then all its sub-scopes become reusing as well. - - - -### `tf.constant_initializer(value=0.0)` {#constant_initializer} +### `tf.constant_initializer(value=0.0, dtype=tf.float32)` {#constant_initializer} Returns an initializer that generates tensors with a single value. @@ -1173,15 +1173,21 @@ Returns an initializer that generates tensors with a single value. * `value`: A Python scalar. All elements of the initialized variable will be set to this value. +* `dtype`: The data type. Only floating point types are supported. ##### Returns: An initializer that generates tensors with a single value. +##### Raises: + + +* `ValueError`: if `dtype` is not a floating point type. + - - - -### `tf.random_normal_initializer(mean=0.0, stddev=1.0, seed=None)` {#random_normal_initializer} +### `tf.random_normal_initializer(mean=0.0, stddev=1.0, seed=None, dtype=tf.float32)` {#random_normal_initializer} Returns an initializer that generates tensors with a normal distribution. @@ -1195,15 +1201,21 @@ Returns an initializer that generates tensors with a normal distribution. * `seed`: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. +* `dtype`: The data type. Only floating point types are supported. ##### Returns: An initializer that generates tensors with a normal distribution. +##### Raises: + + +* `ValueError`: if `dtype` is not a floating point type. + - - - -### `tf.truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None)` {#truncated_normal_initializer} +### `tf.truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None, dtype=tf.float32)` {#truncated_normal_initializer} Returns an initializer that generates a truncated normal distribution. @@ -1222,16 +1234,22 @@ neural network weights and filters. * `seed`: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. +* `dtype`: The data type. Only floating point types are supported. ##### Returns: An initializer that generates tensors with a truncated normal distribution. +##### Raises: + + +* `ValueError`: if `dtype` is not a floating point type. + - - - -### `tf.random_uniform_initializer(minval=0.0, maxval=1.0, seed=None)` {#random_uniform_initializer} +### `tf.random_uniform_initializer(minval=0.0, maxval=1.0, seed=None, dtype=tf.float32)` {#random_uniform_initializer} Returns an initializer that generates tensors with a uniform distribution. @@ -1245,15 +1263,21 @@ Returns an initializer that generates tensors with a uniform distribution. * `seed`: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. +* `dtype`: The data type. Only floating point types are supported. ##### Returns: An initializer that generates tensors with a uniform distribution. +##### Raises: + + +* `ValueError`: if `dtype` is not a floating point type. + - - - -### `tf.uniform_unit_scaling_initializer(factor=1.0, seed=None)` {#uniform_unit_scaling_initializer} +### `tf.uniform_unit_scaling_initializer(factor=1.0, seed=None, dtype=tf.float32)` {#uniform_unit_scaling_initializer} Returns an initializer that generates tensors without scaling variance. @@ -1279,11 +1303,17 @@ numerically computed: for a linear layer it's 1.0, relu: ~1.43, tanh: ~1.15. * `seed`: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. +* `dtype`: The data type. Only floating point types are supported. ##### Returns: An initializer that generates tensors with unit variance. +##### Raises: + + +* `ValueError`: if `dtype` is not a floating point type. + - - - diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py index c0eef1edc80..145058579ea 100644 --- a/tensorflow/python/kernel_tests/init_ops_test.py +++ b/tensorflow/python/kernel_tests/init_ops_test.py @@ -93,87 +93,115 @@ class RandomNormalInitializationTest(tf.test.TestCase): def testInitializerIdentical(self): for use_gpu in [False, True]: - init1 = tf.random_normal_initializer(0.0, 1.0, seed=1) - init2 = tf.random_normal_initializer(0.0, 1.0, seed=1) - self.assertTrue(identicaltest(self, init1, init2, use_gpu)) + for dtype in [tf.float32, tf.float64]: + init1 = tf.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) + init2 = tf.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) + self.assertTrue(identicaltest(self, init1, init2, use_gpu)) def testInitializerDifferent(self): for use_gpu in [False, True]: - init1 = tf.random_normal_initializer(0.0, 1.0, seed=1) - init2 = tf.random_normal_initializer(0.0, 1.0, seed=2) - self.assertFalse(identicaltest(self, init1, init2, use_gpu=use_gpu)) + for dtype in [tf.float32, tf.float64]: + init1 = tf.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) + init2 = tf.random_normal_initializer(0.0, 1.0, seed=2, dtype=dtype) + self.assertFalse(identicaltest(self, init1, init2, use_gpu=use_gpu)) def testDuplicatedInitializer(self): for use_gpu in [False, True]: init = tf.random_normal_initializer(0.0, 1.0) self.assertFalse(duplicated_initializer(self, init, use_gpu, 1)) + def testInvalidDataType(self): + self.assertRaises( + ValueError, + tf.random_normal_initializer, 0.0, 1.0, dtype=tf.string) + class TruncatedNormalInitializationTest(tf.test.TestCase): def testInitializerIdentical(self): for use_gpu in [False, True]: - init1 = tf.truncated_normal_initializer(0.0, 1.0, seed=1) - init2 = tf.truncated_normal_initializer(0.0, 1.0, seed=1) - self.assertTrue(identicaltest(self, init1, init2, use_gpu)) + for dtype in [tf.float32, tf.float64]: + init1 = tf.truncated_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) + init2 = tf.truncated_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) + self.assertTrue(identicaltest(self, init1, init2, use_gpu)) def testInitializerDifferent(self): for use_gpu in [False, True]: - init1 = tf.truncated_normal_initializer(0.0, 1.0, seed=1) - init2 = tf.truncated_normal_initializer(0.0, 1.0, seed=2) - self.assertFalse(identicaltest(self, init1, init2, use_gpu=use_gpu)) + for dtype in [tf.float32, tf.float64]: + init1 = tf.truncated_normal_initializer(0.0, 1.0, seed=1, dtype=dtype) + init2 = tf.truncated_normal_initializer(0.0, 1.0, seed=2, dtype=dtype) + self.assertFalse(identicaltest(self, init1, init2, use_gpu=use_gpu)) def testDuplicatedInitializer(self): for use_gpu in [False, True]: init = tf.truncated_normal_initializer(0.0, 1.0) self.assertFalse(duplicated_initializer(self, init, use_gpu, 1)) + def testInvalidDataType(self): + self.assertRaises( + ValueError, + tf.truncated_normal_initializer, 0.0, 1.0, dtype=tf.string) + class RandomUniformInitializationTest(tf.test.TestCase): def testInitializerIdentical(self): for use_gpu in [False, True]: - init1 = tf.random_uniform_initializer(0.0, 1.0, seed=1) - init2 = tf.random_uniform_initializer(0.0, 1.0, seed=1) - self.assertTrue(identicaltest(self, init1, init2, use_gpu)) + for dtype in [tf.float32, tf.float64]: + init1 = tf.random_uniform_initializer(0.0, 1.0, seed=1, dtype=dtype) + init2 = tf.random_uniform_initializer(0.0, 1.0, seed=1, dtype=dtype) + self.assertTrue(identicaltest(self, init1, init2, use_gpu)) def testInitializerDifferent(self): for use_gpu in [False, True]: - init1 = tf.random_uniform_initializer(0.0, 1.0, seed=1) - init2 = tf.random_uniform_initializer(0.0, 1.0, seed=2) - self.assertFalse(identicaltest(self, init1, init2, use_gpu)) + for dtype in [tf.float32, tf.float64]: + init1 = tf.random_uniform_initializer(0.0, 1.0, seed=1, dtype=dtype) + init2 = tf.random_uniform_initializer(0.0, 1.0, seed=2, dtype=dtype) + self.assertFalse(identicaltest(self, init1, init2, use_gpu)) def testDuplicatedInitializer(self): for use_gpu in [False, True]: init = tf.random_uniform_initializer(0.0, 1.0) self.assertFalse(duplicated_initializer(self, init, use_gpu, 1)) + def testInvalidDataType(self): + self.assertRaises( + ValueError, + tf.random_uniform_initializer, 0.0, 1.0, dtype=tf.string) + class UniformUnitScalingInitializationTest(tf.test.TestCase): def testInitializerIdentical(self): for use_gpu in [False, True]: - init1 = tf.uniform_unit_scaling_initializer(seed=1) - init2 = tf.uniform_unit_scaling_initializer(seed=1) - self.assertTrue(identicaltest(self, init1, init2, use_gpu)) - init3 = tf.uniform_unit_scaling_initializer(1.5, seed=1) - init4 = tf.uniform_unit_scaling_initializer(1.5, seed=1) - self.assertTrue(identicaltest(self, init3, init4, use_gpu)) + for dtype in [tf.float32, tf.float64]: + init1 = tf.uniform_unit_scaling_initializer(seed=1, dtype=dtype) + init2 = tf.uniform_unit_scaling_initializer(seed=1, dtype=dtype) + self.assertTrue(identicaltest(self, init1, init2, use_gpu)) + init3 = tf.uniform_unit_scaling_initializer(1.5, seed=1, dtype=dtype) + init4 = tf.uniform_unit_scaling_initializer(1.5, seed=1, dtype=dtype) + self.assertTrue(identicaltest(self, init3, init4, use_gpu)) def testInitializerDifferent(self): for use_gpu in [False, True]: - init1 = tf.uniform_unit_scaling_initializer(seed=1) - init2 = tf.uniform_unit_scaling_initializer(seed=2) - init3 = tf.uniform_unit_scaling_initializer(1.5, seed=1) - self.assertFalse(identicaltest(self, init1, init2, use_gpu)) - self.assertFalse(identicaltest(self, init1, init3, use_gpu)) - self.assertFalse(identicaltest(self, init2, init3, use_gpu)) + for dtype in [tf.float32, tf.float64]: + init1 = tf.uniform_unit_scaling_initializer(seed=1, dtype=dtype) + init2 = tf.uniform_unit_scaling_initializer(seed=2, dtype=dtype) + init3 = tf.uniform_unit_scaling_initializer(1.5, seed=1, dtype=dtype) + self.assertFalse(identicaltest(self, init1, init2, use_gpu)) + self.assertFalse(identicaltest(self, init1, init3, use_gpu)) + self.assertFalse(identicaltest(self, init2, init3, use_gpu)) def testDuplicatedInitializer(self): for use_gpu in [False, True]: init = tf.uniform_unit_scaling_initializer() self.assertFalse(duplicated_initializer(self, init, use_gpu, 1)) + def testInvalidDataType(self): + self.assertRaises( + ValueError, + tf.uniform_unit_scaling_initializer, dtype=tf.string) + class RandomWalkShapeTest(tf.test.TestCase): diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py index 8c52a232cb2..eda688b1a9e 100644 --- a/tensorflow/python/ops/init_ops.py +++ b/tensorflow/python/ops/init_ops.py @@ -27,22 +27,46 @@ from tensorflow.python.ops import nn_ops from tensorflow.python.ops import random_ops -# TODO(mrry): PEP8 these. -def constant_initializer(value=0.0): +def _assert_float_dtype(dtype): + """Validate and return floating point type based on `dtype`. + + `dtype` must be a floating point type. + + Args: + dtype: The data type to validate. + + Returns: + Validated type. + + Raises: + ValueError: if `dtype` is not a floating point type. + """ + if not dtype.is_floating: + raise ValueError("Expected floating point type, got %s." % dtype) + return dtype + + +def constant_initializer(value=0.0, dtype=dtypes.float32): """Returns an initializer that generates tensors with a single value. Args: value: A Python scalar. All elements of the initialized variable will be set to this value. + dtype: The data type. Only floating point types are supported. Returns: An initializer that generates tensors with a single value. + + Raises: + ValueError: if `dtype` is not a floating point type. """ - def _initializer(shape, dtype=dtypes.float32): + def _initializer(shape, dtype=_assert_float_dtype(dtype)): return constant_op.constant(value, dtype=dtype, shape=shape) return _initializer -def random_uniform_initializer(minval=0.0, maxval=1.0, seed=None): + +def random_uniform_initializer(minval=0.0, maxval=1.0, seed=None, + dtype=dtypes.float32): """Returns an initializer that generates tensors with a uniform distribution. Args: @@ -53,15 +77,21 @@ def random_uniform_initializer(minval=0.0, maxval=1.0, seed=None): seed: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. + dtype: The data type. Only floating point types are supported. Returns: An initializer that generates tensors with a uniform distribution. + + Raises: + ValueError: if `dtype` is not a floating point type. """ - def _initializer(shape, dtype=dtypes.float32): + def _initializer(shape, dtype=_assert_float_dtype(dtype)): return random_ops.random_uniform(shape, minval, maxval, dtype, seed=seed) return _initializer -def random_normal_initializer(mean=0.0, stddev=1.0, seed=None): + +def random_normal_initializer(mean=0.0, stddev=1.0, seed=None, + dtype=dtypes.float32): """Returns an initializer that generates tensors with a normal distribution. Args: @@ -72,15 +102,21 @@ def random_normal_initializer(mean=0.0, stddev=1.0, seed=None): seed: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. + dtype: The data type. Only floating point types are supported. Returns: An initializer that generates tensors with a normal distribution. + + Raises: + ValueError: if `dtype` is not a floating point type. """ - def _initializer(shape, dtype=dtypes.float32): + def _initializer(shape, dtype=_assert_float_dtype(dtype)): return random_ops.random_normal(shape, mean, stddev, dtype, seed=seed) return _initializer -def truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None): + +def truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None, + dtype=dtypes.float32): """Returns an initializer that generates a truncated normal distribution. These values are similar to values from a `random_normal_initializer` @@ -96,16 +132,22 @@ def truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None): seed: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. + dtype: The data type. Only floating point types are supported. Returns: An initializer that generates tensors with a truncated normal distribution. + + Raises: + ValueError: if `dtype` is not a floating point type. """ - def _initializer(shape, dtype=dtypes.float32): + def _initializer(shape, dtype=_assert_float_dtype(dtype)): return random_ops.truncated_normal(shape, mean, stddev, dtype, seed=seed) return _initializer -def uniform_unit_scaling_initializer(factor=1.0, seed=None): + +def uniform_unit_scaling_initializer(factor=1.0, seed=None, + dtype=dtypes.float32): """Returns an initializer that generates tensors without scaling variance. When initializing a deep network, it is in principle advantageous to keep @@ -128,11 +170,15 @@ def uniform_unit_scaling_initializer(factor=1.0, seed=None): seed: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. + dtype: The data type. Only floating point types are supported. Returns: An initializer that generates tensors with unit variance. + + Raises: + ValueError: if `dtype` is not a floating point type. """ - def _initializer(shape, dtype=dtypes.float32): + def _initializer(shape, dtype=_assert_float_dtype(dtype)): input_size = 1.0 # Estimating input size is not possible to do perfectly, but we try. # The estimate, obtained by multiplying all dimensions but the last one,