experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update init_ops to also support initialization of float64 variables.

Browse Source 
Change: 111625948
This commit is contained in:
Yutaka Leon 2016-01-07 13:26:35 -08:00 committed by Vijay Vasudevan
parent ca47376b3c
commit 4c4de46cf2
3 changed files with 150 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
tensorflow/g3doc/api_docs/python/state_ops.md
View File

@ -1164,7 +1164,7 @@ then all its sub-scopes become reusing as well.
- - -
### `tf.constant_initializer(value=0.0)` {#constant_initializer}
### `tf.constant_initializer(value=0.0, dtype=tf.float32)` {#constant_initializer}
Returns an initializer that generates tensors with a single value.
@ -1173,15 +1173,21 @@ Returns an initializer that generates tensors with a single value.
* <b>`value`</b>: A Python scalar. All elements of the initialized variable
will be set to this value.
* <b>`dtype`</b>: The data type. Only floating point types are supported.
##### Returns:
An initializer that generates tensors with a single value.
##### Raises:
* <b>`ValueError`</b>: if `dtype` is not a floating point type.
- - -
### `tf.random_normal_initializer(mean=0.0, stddev=1.0, seed=None)` {#random_normal_initializer}
### `tf.random_normal_initializer(mean=0.0, stddev=1.0, seed=None, dtype=tf.float32)` {#random_normal_initializer}
Returns an initializer that generates tensors with a normal distribution.
@ -1195,15 +1201,21 @@ Returns an initializer that generates tensors with a normal distribution.
* <b>`seed`</b>: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
* <b>`dtype`</b>: The data type. Only floating point types are supported.
##### Returns:
An initializer that generates tensors with a normal distribution.
##### Raises:
* <b>`ValueError`</b>: if `dtype` is not a floating point type.
- - -
### `tf.truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None)` {#truncated_normal_initializer}
### `tf.truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None, dtype=tf.float32)` {#truncated_normal_initializer}
Returns an initializer that generates a truncated normal distribution.
@ -1222,16 +1234,22 @@ neural network weights and filters.
* <b>`seed`</b>: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
* <b>`dtype`</b>: The data type. Only floating point types are supported.
##### Returns:
An initializer that generates tensors with a truncated normal
distribution.
##### Raises:
* <b>`ValueError`</b>: if `dtype` is not a floating point type.
- - -
### `tf.random_uniform_initializer(minval=0.0, maxval=1.0, seed=None)` {#random_uniform_initializer}
### `tf.random_uniform_initializer(minval=0.0, maxval=1.0, seed=None, dtype=tf.float32)` {#random_uniform_initializer}
Returns an initializer that generates tensors with a uniform distribution.
@ -1245,15 +1263,21 @@ Returns an initializer that generates tensors with a uniform distribution.
* <b>`seed`</b>: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
* <b>`dtype`</b>: The data type. Only floating point types are supported.
##### Returns:
An initializer that generates tensors with a uniform distribution.
##### Raises:
* <b>`ValueError`</b>: if `dtype` is not a floating point type.
- - -
### `tf.uniform_unit_scaling_initializer(factor=1.0, seed=None)` {#uniform_unit_scaling_initializer}
### `tf.uniform_unit_scaling_initializer(factor=1.0, seed=None, dtype=tf.float32)` {#uniform_unit_scaling_initializer}
Returns an initializer that generates tensors without scaling variance.
@ -1279,11 +1303,17 @@ numerically computed: for a linear layer it's 1.0, relu: ~1.43, tanh: ~1.15.
* <b>`seed`</b>: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
* <b>`dtype`</b>: The data type. Only floating point types are supported.
##### Returns:
An initializer that generates tensors with unit variance.
##### Raises:
* <b>`ValueError`</b>: if `dtype` is not a floating point type.
- - -

88
tensorflow/python/kernel_tests/init_ops_test.py
View File

@ -93,87 +93,115 @@ class RandomNormalInitializationTest(tf.test.TestCase):
def testInitializerIdentical(self):
for use_gpu in [False, True]:
init1 = tf.random_normal_initializer(0.0, 1.0, seed=1)
init2 = tf.random_normal_initializer(0.0, 1.0, seed=1)
self.assertTrue(identicaltest(self, init1, init2, use_gpu))
for dtype in [tf.float32, tf.float64]:
init1 = tf.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
init2 = tf.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
self.assertTrue(identicaltest(self, init1, init2, use_gpu))
def testInitializerDifferent(self):
for use_gpu in [False, True]:
init1 = tf.random_normal_initializer(0.0, 1.0, seed=1)
init2 = tf.random_normal_initializer(0.0, 1.0, seed=2)
self.assertFalse(identicaltest(self, init1, init2, use_gpu=use_gpu))
for dtype in [tf.float32, tf.float64]:
init1 = tf.random_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
init2 = tf.random_normal_initializer(0.0, 1.0, seed=2, dtype=dtype)
self.assertFalse(identicaltest(self, init1, init2, use_gpu=use_gpu))
def testDuplicatedInitializer(self):
for use_gpu in [False, True]:
init = tf.random_normal_initializer(0.0, 1.0)
self.assertFalse(duplicated_initializer(self, init, use_gpu, 1))
def testInvalidDataType(self):
self.assertRaises(
ValueError,
tf.random_normal_initializer, 0.0, 1.0, dtype=tf.string)
class TruncatedNormalInitializationTest(tf.test.TestCase):
def testInitializerIdentical(self):
for use_gpu in [False, True]:
init1 = tf.truncated_normal_initializer(0.0, 1.0, seed=1)
init2 = tf.truncated_normal_initializer(0.0, 1.0, seed=1)
self.assertTrue(identicaltest(self, init1, init2, use_gpu))
for dtype in [tf.float32, tf.float64]:
init1 = tf.truncated_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
init2 = tf.truncated_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
self.assertTrue(identicaltest(self, init1, init2, use_gpu))
def testInitializerDifferent(self):
for use_gpu in [False, True]:
init1 = tf.truncated_normal_initializer(0.0, 1.0, seed=1)
init2 = tf.truncated_normal_initializer(0.0, 1.0, seed=2)
self.assertFalse(identicaltest(self, init1, init2, use_gpu=use_gpu))
for dtype in [tf.float32, tf.float64]:
init1 = tf.truncated_normal_initializer(0.0, 1.0, seed=1, dtype=dtype)
init2 = tf.truncated_normal_initializer(0.0, 1.0, seed=2, dtype=dtype)
self.assertFalse(identicaltest(self, init1, init2, use_gpu=use_gpu))
def testDuplicatedInitializer(self):
for use_gpu in [False, True]:
init = tf.truncated_normal_initializer(0.0, 1.0)
self.assertFalse(duplicated_initializer(self, init, use_gpu, 1))
def testInvalidDataType(self):
self.assertRaises(
ValueError,
tf.truncated_normal_initializer, 0.0, 1.0, dtype=tf.string)
class RandomUniformInitializationTest(tf.test.TestCase):
def testInitializerIdentical(self):
for use_gpu in [False, True]:
init1 = tf.random_uniform_initializer(0.0, 1.0, seed=1)
init2 = tf.random_uniform_initializer(0.0, 1.0, seed=1)
self.assertTrue(identicaltest(self, init1, init2, use_gpu))
for dtype in [tf.float32, tf.float64]:
init1 = tf.random_uniform_initializer(0.0, 1.0, seed=1, dtype=dtype)
init2 = tf.random_uniform_initializer(0.0, 1.0, seed=1, dtype=dtype)
self.assertTrue(identicaltest(self, init1, init2, use_gpu))
def testInitializerDifferent(self):
for use_gpu in [False, True]:
init1 = tf.random_uniform_initializer(0.0, 1.0, seed=1)
init2 = tf.random_uniform_initializer(0.0, 1.0, seed=2)
self.assertFalse(identicaltest(self, init1, init2, use_gpu))
for dtype in [tf.float32, tf.float64]:
init1 = tf.random_uniform_initializer(0.0, 1.0, seed=1, dtype=dtype)
init2 = tf.random_uniform_initializer(0.0, 1.0, seed=2, dtype=dtype)
self.assertFalse(identicaltest(self, init1, init2, use_gpu))
def testDuplicatedInitializer(self):
for use_gpu in [False, True]:
init = tf.random_uniform_initializer(0.0, 1.0)
self.assertFalse(duplicated_initializer(self, init, use_gpu, 1))
def testInvalidDataType(self):
self.assertRaises(
ValueError,
tf.random_uniform_initializer, 0.0, 1.0, dtype=tf.string)
class UniformUnitScalingInitializationTest(tf.test.TestCase):
def testInitializerIdentical(self):
for use_gpu in [False, True]:
init1 = tf.uniform_unit_scaling_initializer(seed=1)
init2 = tf.uniform_unit_scaling_initializer(seed=1)
self.assertTrue(identicaltest(self, init1, init2, use_gpu))
init3 = tf.uniform_unit_scaling_initializer(1.5, seed=1)
init4 = tf.uniform_unit_scaling_initializer(1.5, seed=1)
self.assertTrue(identicaltest(self, init3, init4, use_gpu))
for dtype in [tf.float32, tf.float64]:
init1 = tf.uniform_unit_scaling_initializer(seed=1, dtype=dtype)
init2 = tf.uniform_unit_scaling_initializer(seed=1, dtype=dtype)
self.assertTrue(identicaltest(self, init1, init2, use_gpu))
init3 = tf.uniform_unit_scaling_initializer(1.5, seed=1, dtype=dtype)
init4 = tf.uniform_unit_scaling_initializer(1.5, seed=1, dtype=dtype)
self.assertTrue(identicaltest(self, init3, init4, use_gpu))
def testInitializerDifferent(self):
for use_gpu in [False, True]:
init1 = tf.uniform_unit_scaling_initializer(seed=1)
init2 = tf.uniform_unit_scaling_initializer(seed=2)
init3 = tf.uniform_unit_scaling_initializer(1.5, seed=1)
self.assertFalse(identicaltest(self, init1, init2, use_gpu))
self.assertFalse(identicaltest(self, init1, init3, use_gpu))
self.assertFalse(identicaltest(self, init2, init3, use_gpu))
for dtype in [tf.float32, tf.float64]:
init1 = tf.uniform_unit_scaling_initializer(seed=1, dtype=dtype)
init2 = tf.uniform_unit_scaling_initializer(seed=2, dtype=dtype)
init3 = tf.uniform_unit_scaling_initializer(1.5, seed=1, dtype=dtype)
self.assertFalse(identicaltest(self, init1, init2, use_gpu))
self.assertFalse(identicaltest(self, init1, init3, use_gpu))
self.assertFalse(identicaltest(self, init2, init3, use_gpu))
def testDuplicatedInitializer(self):
for use_gpu in [False, True]:
init = tf.uniform_unit_scaling_initializer()
self.assertFalse(duplicated_initializer(self, init, use_gpu, 1))
def testInvalidDataType(self):
self.assertRaises(
ValueError,
tf.uniform_unit_scaling_initializer, dtype=tf.string)
class RandomWalkShapeTest(tf.test.TestCase):

68
tensorflow/python/ops/init_ops.py
View File

@ -27,22 +27,46 @@ from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import random_ops
# TODO(mrry): PEP8 these.
def constant_initializer(value=0.0):
def _assert_float_dtype(dtype):
"""Validate and return floating point type based on `dtype`.
`dtype` must be a floating point type.
Args:
dtype: The data type to validate.
Returns:
Validated type.
Raises:
ValueError: if `dtype` is not a floating point type.
"""
if not dtype.is_floating:
raise ValueError("Expected floating point type, got %s." % dtype)
return dtype
def constant_initializer(value=0.0, dtype=dtypes.float32):
"""Returns an initializer that generates tensors with a single value.
Args:
value: A Python scalar. All elements of the initialized variable
will be set to this value.
dtype: The data type. Only floating point types are supported.
Returns:
An initializer that generates tensors with a single value.
Raises:
ValueError: if `dtype` is not a floating point type.
"""
def _initializer(shape, dtype=dtypes.float32):
def _initializer(shape, dtype=_assert_float_dtype(dtype)):
return constant_op.constant(value, dtype=dtype, shape=shape)
return _initializer
def random_uniform_initializer(minval=0.0, maxval=1.0, seed=None):
def random_uniform_initializer(minval=0.0, maxval=1.0, seed=None,
dtype=dtypes.float32):
"""Returns an initializer that generates tensors with a uniform distribution.
Args:
@ -53,15 +77,21 @@ def random_uniform_initializer(minval=0.0, maxval=1.0, seed=None):
seed: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
dtype: The data type. Only floating point types are supported.
Returns:
An initializer that generates tensors with a uniform distribution.
Raises:
ValueError: if `dtype` is not a floating point type.
"""
def _initializer(shape, dtype=dtypes.float32):
def _initializer(shape, dtype=_assert_float_dtype(dtype)):
return random_ops.random_uniform(shape, minval, maxval, dtype, seed=seed)
return _initializer
def random_normal_initializer(mean=0.0, stddev=1.0, seed=None):
def random_normal_initializer(mean=0.0, stddev=1.0, seed=None,
dtype=dtypes.float32):
"""Returns an initializer that generates tensors with a normal distribution.
Args:
@ -72,15 +102,21 @@ def random_normal_initializer(mean=0.0, stddev=1.0, seed=None):
seed: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
dtype: The data type. Only floating point types are supported.
Returns:
An initializer that generates tensors with a normal distribution.
Raises:
ValueError: if `dtype` is not a floating point type.
"""
def _initializer(shape, dtype=dtypes.float32):
def _initializer(shape, dtype=_assert_float_dtype(dtype)):
return random_ops.random_normal(shape, mean, stddev, dtype, seed=seed)
return _initializer
def truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None):
def truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None,
dtype=dtypes.float32):
"""Returns an initializer that generates a truncated normal distribution.
These values are similar to values from a `random_normal_initializer`
@ -96,16 +132,22 @@ def truncated_normal_initializer(mean=0.0, stddev=1.0, seed=None):
seed: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
dtype: The data type. Only floating point types are supported.
Returns:
An initializer that generates tensors with a truncated normal
distribution.
Raises:
ValueError: if `dtype` is not a floating point type.
"""
def _initializer(shape, dtype=dtypes.float32):
def _initializer(shape, dtype=_assert_float_dtype(dtype)):
return random_ops.truncated_normal(shape, mean, stddev, dtype, seed=seed)
return _initializer
def uniform_unit_scaling_initializer(factor=1.0, seed=None):
def uniform_unit_scaling_initializer(factor=1.0, seed=None,
dtype=dtypes.float32):
"""Returns an initializer that generates tensors without scaling variance.
When initializing a deep network, it is in principle advantageous to keep
@ -128,11 +170,15 @@ def uniform_unit_scaling_initializer(factor=1.0, seed=None):
seed: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
dtype: The data type. Only floating point types are supported.
Returns:
An initializer that generates tensors with unit variance.
Raises:
ValueError: if `dtype` is not a floating point type.
"""
def _initializer(shape, dtype=dtypes.float32):
def _initializer(shape, dtype=_assert_float_dtype(dtype)):
input_size = 1.0
# Estimating input size is not possible to do perfectly, but we try.
# The estimate, obtained by multiplying all dimensions but the last one,