experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove "infer" dtype policy.

Browse Source 
It has been renamed to "_infer" and no longer is intended to be used by users. The documentation referring to it has been removed from the Policy docstring. I still have to keep it, since it is the default if a user disables V2 behavior with tf.compat.v1.disable_v2_behavior(). I documented this in the global_policy() docstring.

Since the only available Policy in TF1 was "infer", Policy is now unusable in TF1, so I now expose it in TF2 only. Even before this change, Policy was useless in TF1, since the default was already "infer".

PiperOrigin-RevId: 299918175
Change-Id: Iddd7acfb367ef69181e88d0933c7b5e657583db1
This commit is contained in:
Reed Wanderman-Milne 2020-03-09 13:22:22 -07:00 committed by TensorFlower Gardener
parent 1fa7c80fe0
commit 2d51302640
9 changed files with 73 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tensorflow/python/keras/engine/base_layer.py
View File

@ -508,7 +508,7 @@ class Layer(module.Module, version_utils.LayerVersionSelector):
dtype = self.dtype or backend.floatx()
dtype = dtypes.as_dtype(dtype)
if self._dtype_policy.variable_dtype is None:
# The policy is "infer", so we infer the policy from the variable dtype.
# The policy is "_infer", so we infer the policy from the variable dtype.
self._dtype_policy = policy.Policy(dtype.base_dtype.name)
initializer = initializers.get(initializer)
regularizer = regularizers.get(regularizer)

2
tensorflow/python/keras/engine/base_layer_v1.py
View File

@ -378,7 +378,7 @@ class Layer(base_layer.Layer):
dtype = self.dtype or backend.floatx()
dtype = dtypes.as_dtype(dtype)
if self._dtype_policy.variable_dtype is None:
# The policy is "infer", so we infer the policy from the variable dtype.
# The policy is "_infer", so we infer the policy from the variable dtype.
self._dtype_policy = policy.Policy(dtype.base_dtype.name)
initializer = initializers.get(initializer)
regularizer = regularizers.get(regularizer)

2
tensorflow/python/keras/mixed_precision/experimental/get_layer_policy.py
View File

@ -25,7 +25,7 @@ from tensorflow.python.keras.engine import base_layer
from tensorflow.python.util.tf_export import keras_export
@keras_export('keras.mixed_precision.experimental.get_layer_policy')
@keras_export('keras.mixed_precision.experimental.get_layer_policy', v1=[])
def get_layer_policy(layer):
"""Returns the dtype policy of a layer.

14
tensorflow/python/keras/mixed_precision/experimental/keras_test.py
View File

@ -371,24 +371,24 @@ class KerasLayerTest(keras_parameterized.TestCase):
self.assertEqual(layer(x).dtype, 'float64')
self.assertEqual(layer.v.dtype, 'float64')
layer = mp_test_util.MultiplyLayer(dtype=policy.Policy('infer'))
layer = mp_test_util.MultiplyLayer(dtype=policy.Policy('_infer'))
config = layer.get_config()
self.assertIsNone(config['dtype'])
layer = mp_test_util.MultiplyLayer.from_config(config)
# If a layer is serialized with the "infer" policy, when deserialized into
# TF 2 it will have the global policy instead of "infer". This is because
# "infer" is serialized into None, and passing dtype=None in TensorFlow 2
# indicates to use the global policy.
# If a layer is serialized with the "_infer" policy, when deserialized
# into TF 2 it will have the global policy instead of "_infer". This is
# because "_infer" is serialized into None, and passing dtype=None in
# TensorFlow 2 indicates to use the global policy.
self.assertEqual(layer.dtype, 'float32')
self.assertEqual(layer(x).dtype, 'float32')
self.assertEqual(layer.v.dtype, 'float32')
layer = mp_test_util.MultiplyLayer(dtype=policy.Policy('infer',
layer = mp_test_util.MultiplyLayer(dtype=policy.Policy('_infer',
loss_scale=2.))
config = layer.get_config()
self.assertEqual(config['dtype'],
{'class_name': 'Policy',
'config': {'name': 'infer',
'config': {'name': '_infer',
'loss_scale': {
'class_name': 'FixedLossScale',
'config': {'loss_scale_value': 2.0}}}})

80
tensorflow/python/keras/mixed_precision/experimental/policy.py
View File

@ -37,7 +37,7 @@ from tensorflow.python.util.tf_export import keras_export
USE_DEFAULT = 'USE_DEFAULT'
@keras_export('keras.mixed_precision.experimental.Policy')
@keras_export('keras.mixed_precision.experimental.Policy', v1=[])
class Policy(object):
"""A dtype policy for a Keras layer.
@ -287,16 +287,6 @@ class Policy(object):
If you did not pass `dtype=inputs.dtype` to `tf.random.normal`, a `TypeError`
would have occurred. This is because the dtype defaults to `"float32"`, so the
layer would only work if the inputs were float32.
### The deprecated "infer" policy
In addition to the above mentioned policies, a policy can also be "infer".
This Policy is deprecated, and it is not recommended. When a layer has an
infer policy, it will infer the computation and variable dtype from the first
input the first time the layer is called. Once the layer is called for the
first time, the layer's policy will change to the dtype of the first input.
In TensorFlow 1, only the "infer" policy is available.
"""
def __init__(self, name, loss_scale=USE_DEFAULT):
@ -315,14 +305,12 @@ class Policy(object):
bfloat16, while the variable dtype is float32. With 'mixed_float16',
a dynamic loss scale is used. These policies are used for mixed
precision training.
* 'infer' (deprecated): Infer the compute and variable dtype from the
input dtype.
loss_scale: A `tf.mixed_precision.experimental.LossScale`, an int (which
uses a `FixedLossScale`), or the string "dynamic" (which uses a
`DynamicLossScale`). Defaults to using no loss scaling unless `name` is
"mixed_float16", in which case this defaults to "dynamic". Only
`tf.keras.Model`s, not layers, use the loss scale, and it is only used
during `Model.fit`, `Model.train_on_batch`, and other similar methods.
uses a `FixedLossScale`), or the string "dynamic" (which uses a
`DynamicLossScale`). Defaults to using no loss scaling unless `name` is
"mixed_float16", in which case this defaults to "dynamic". Only
`tf.keras.Model`s, not layers, use the loss scale, and it is only used
during `Model.fit`, `Model.train_on_batch`, and other similar methods.
"""
if isinstance(name, dtypes.DType):
raise TypeError("'name' must be a string, not a DType. "
@ -373,7 +361,17 @@ class Policy(object):
return 'float16', 'float32'
elif name == 'mixed_bfloat16':
return 'bfloat16', 'float32'
elif name == 'infer':
elif name == '_infer':
# The "_infer" policy exists only for compatibility with TF 1, where
# "_infer" is the default. The behavior matches the behavior of TF 1's
# behavior before policies were introduced. With "_infer", the computation
# and variable dtype are inferred from the first input the first time the
# layer is called. Once the layer is called for the first time, the
# layer's policy will change to the dtype of the first input, and it will
# no longer have the "_infer" policy.
#
# The infer policy should be considered an implementation detail and may
# be removed in the future.
return None, None
try:
@ -397,8 +395,7 @@ class Policy(object):
avoid type errors.
Returns:
The variable dtype of this policy, or None if the variable dtype should be
inferred from the inputs.
The variable dtype of this policy.
"""
return self._variable_dtype
@ -425,8 +422,7 @@ class Policy(object):
keeping intermediate computations in float32.
Returns:
The compute dtype of this policy, or None if the compute dtype should be
inferred from the inputs.
The compute dtype of this policy.
"""
return self._compute_dtype
@ -480,22 +476,28 @@ class Policy(object):
# The current global policy in effect. If None, it means the current value of
# floatx should be used as the policy if the V2 dtype behavior is enabled,
# or "infer" otherwise.
# or "_infer" otherwise.
# TODO(reedwm): Make this thread local?
_global_policy = None
@keras_export('keras.mixed_precision.experimental.global_policy')
@keras_export('keras.mixed_precision.experimental.global_policy', v1=[])
def global_policy():
"""Returns the global Policy.
The global policy is the default policy used for layers, if no policy is
passed to the layer constructor. If no policy has been set with
`keras.mixed_precision.experimental.set_policy`, this will return a policy
constructed from `tf.keras.backend.floatx()` in TensorFlow 2 (floatx defaults
to float32), or an "infer" policy in TensorFlow 1.
constructed from `tf.keras.backend.floatx()` (floatx defaults to float32).
See `keras.mixed_precision.experimental.Policy` for more information.
If TensorFlow 2 behavior has been disabled with
`tf.compat.v1.disable_v2_behavior()`, this will instead return a special
"_infer" policy which infers the dtype from the dtype of the first input the
first time the layer is called. This behavior matches the behavior that
existed in TensorFlow 1.
See `tf.keras.mixed_precision.experimental.Policy` for more information on
policies.
Returns:
The global Policy.
@ -504,7 +506,7 @@ def global_policy():
if base_layer_utils.v2_dtype_behavior_enabled():
return Policy(backend.floatx())
else:
return Policy('infer')
return Policy('_infer')
return _global_policy
@ -530,14 +532,13 @@ def _check_if_mixed_precision_graph_rewrite_is_enabled():
'customizable.')
@keras_export('keras.mixed_precision.experimental.set_policy')
@keras_export('keras.mixed_precision.experimental.set_policy', v1=[])
def set_policy(policy):
"""Sets the global Policy.
The global policy is the default policy used for layers, if no policy is
passed to the layer constructor. If no global policy is set, layers will
instead default to a Policy constructed from `tf.keras.backend.floatx()` in
TensorFlow 2. In TensorFlow 1, layers default to an "infer" policy.
instead default to a Policy constructed from `tf.keras.backend.floatx()`.
See `keras.mixed_precision.experimental.Policy` for more information.
@ -546,13 +547,10 @@ def set_policy(policy):
"""
global _global_policy
_check_if_mixed_precision_graph_rewrite_is_enabled()
if not base_layer_utils.v2_dtype_behavior_enabled():
raise ValueError('The global policy can only be set in TensorFlow 2')
if policy is not None and not isinstance(policy, Policy):
policy = Policy(policy)
if (policy and not base_layer_utils.v2_dtype_behavior_enabled() and
policy.compute_dtype):
raise ValueError(
'The global policy can only be set to a non-infer policy in TensorFlow '
'2')
_global_policy = policy
mixed_precision_global_state.using_default_mixed_precision_policy = (
_global_policy is None)
@ -592,7 +590,7 @@ def _policy_equivalent_to_dtype(policy):
dtypes are the same and the policy does not cause the layer/model to have
additional behavior, such as loss scaling.
The "infer" policy is considered equivalent to a single dtype.
The "_infer" policy is considered equivalent to a single dtype.
Args:
policy: A Policy.
@ -604,7 +602,7 @@ def _policy_equivalent_to_dtype(policy):
# equivalent to a dtype.
return (type(policy) == Policy and # pylint: disable=unidiomatic-typecheck
list(policy.get_config().keys()) == ['name'] and
(policy.name == 'infer' or _is_convertible_to_dtype(policy.name)))
(policy.name == '_infer' or _is_convertible_to_dtype(policy.name)))
def serialize(policy):
@ -612,7 +610,7 @@ def serialize(policy):
# We return either None or the policy name for compatibility with older
# versions of Keras. If the policy name is returned, it is a dtype string
# such as 'float32'.
return None if policy.name == 'infer' else policy.name
return None if policy.name == '_infer' else policy.name
return generic_utils.serialize_keras_object(policy)
@ -620,7 +618,7 @@ def deserialize(config, custom_objects=None):
if isinstance(config, str) and _is_convertible_to_dtype(config):
return Policy(config)
if config is None:
return Policy('infer')
return Policy('_infer')
module_objects = {'Policy': Policy}
return generic_utils.deserialize_keras_object(
config,

43
tensorflow/python/keras/mixed_precision/experimental/policy_test.py
View File

@ -39,10 +39,6 @@ class PolicyTest(test.TestCase):
@testing_utils.enable_v2_dtype_behavior
def test_dtype_attributes(self):
policy = mp_policy.Policy('infer')
self.assertEqual(policy.compute_dtype, None)
self.assertEqual(policy.variable_dtype, None)
for dtype in 'int32', 'bool', 'float16', 'float32':
policy = mp_policy.Policy(dtype)
self.assertEqual(policy.name, dtype)
@ -55,9 +51,13 @@ class PolicyTest(test.TestCase):
self.assertEqual(policy.compute_dtype, dtype)
self.assertEqual(policy.variable_dtype, 'float32')
policy = mp_policy.Policy('_infer')
self.assertEqual(policy.compute_dtype, None)
self.assertEqual(policy.variable_dtype, None)
@testing_utils.enable_v2_dtype_behavior
def test_repr(self):
for policy in ('infer', 'float32', 'int8', 'mixed_bfloat16'):
for policy in ('float32', 'int8', 'mixed_bfloat16', '_infer'):
self.assertEqual(repr(mp_policy.Policy(policy)),
'<Policy "%s", loss_scale=None>' % policy)
self.assertEqual(repr(mp_policy.Policy('float16', loss_scale=2)),
@ -137,15 +137,15 @@ class PolicyTest(test.TestCase):
if base_layer_utils.v2_dtype_behavior_enabled():
default_policy = 'float32'
else:
default_policy = 'infer'
default_policy = '_infer'
self.assertEqual(mp_policy.global_policy().name, default_policy)
try:
mp_policy.set_policy('mixed_float16')
self.assertEqual(mp_policy.global_policy().name, 'mixed_float16')
with ops.Graph().as_default(): # Policies are not associated with a graph
self.assertEqual(mp_policy.global_policy().name, 'mixed_float16')
mp_policy.set_policy('infer')
self.assertEqual(mp_policy.global_policy().name, 'infer')
mp_policy.set_policy('_infer')
self.assertEqual(mp_policy.global_policy().name, '_infer')
policy = mp_policy.Policy('mixed_bfloat16')
mp_policy.set_policy(policy)
self.assertIs(mp_policy.global_policy(), policy)
@ -196,11 +196,11 @@ class PolicyTest(test.TestCase):
if base_layer_utils.v2_dtype_behavior_enabled():
default_policy = 'float32'
else:
default_policy = 'infer'
default_policy = '_infer'
with mp_policy.policy_scope('mixed_float16'):
self.assertEqual(mp_policy.global_policy().name, 'mixed_float16')
with mp_policy.policy_scope('infer'):
self.assertEqual(mp_policy.global_policy().name, 'infer')
with mp_policy.policy_scope('_infer'):
self.assertEqual(mp_policy.global_policy().name, '_infer')
self.assertEqual(mp_policy.global_policy().name, 'mixed_float16')
self.assertEqual(mp_policy.global_policy().name, default_policy)
@ -212,7 +212,7 @@ class PolicyTest(test.TestCase):
mp_policy.Policy('int16'),
mp_policy.Policy('mixed_float16'),
mp_policy.Policy('mixed_bfloat16'),
mp_policy.Policy('infer'),
mp_policy.Policy('_infer'),
mp_policy.Policy('float32', loss_scale=2.),
mp_policy.Policy('float32', loss_scale=None),
mp_policy.Policy('mixed_float16', loss_scale=2.),
@ -236,8 +236,8 @@ class PolicyTest(test.TestCase):
new_policy = mp_policy.deserialize(config)
self.assertEqual(str(policy), str(new_policy))
# Test "infer" policy
policy = mp_policy.Policy('infer')
# Test "_infer" policy
policy = mp_policy.Policy('_infer')
config = mp_policy.serialize(policy)
self.assertIsNone(config)
new_policy = mp_policy.deserialize(config)
@ -301,19 +301,20 @@ class PolicyTest(test.TestCase):
@testing_utils.disable_v2_dtype_behavior
def test_v1_dtype_behavior(self):
# Only the "infer" policy is allowed with V1 dtype behavior
with mp_policy.policy_scope(mp_policy.Policy('infer')):
pass
# Non-infer policies are not allowed with V1 dtype behavior
# Setting global policies are not allowed with V1 dtype behavior
with self.assertRaisesRegexp(
ValueError,
'global policy can only be set to a non-infer policy in TensorFlow 2'):
'global policy can only be set in TensorFlow 2'):
with mp_policy.policy_scope(mp_policy.Policy('_infer')):
pass
with self.assertRaisesRegexp(
ValueError,
'global policy can only be set in TensorFlow 2'):
with mp_policy.policy_scope(mp_policy.Policy('float32')):
pass
with self.assertRaisesRegexp(
ValueError,
'global policy can only be set to a non-infer policy in TensorFlow 2'):
'global policy can only be set in TensorFlow 2'):
with mp_policy.policy_scope(mp_policy.Policy('mixed_float16')):
pass

4
tensorflow/python/layers/base.py
View File

@ -203,8 +203,8 @@ class Layer(base_layer.Layer):
if dtype is None:
# Indicates to infer dtype from inputs. When the V2 dtype behavior is
# enabled, Keras layers default their dtype to floatx instead, so we pass
# an "infer" policy to keep the old V1 behavior.
dtype = policy.Policy('infer')
# an "_infer" policy to keep the old V1 behavior.
dtype = policy.Policy('_infer')
if 'autocast' not in kwargs:
kwargs['autocast'] = False

37
tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-policy.pbtxt
View File

@ -1,37 +0,0 @@
path: "tensorflow.keras.mixed_precision.experimental.Policy"
tf_class {
is_instance: "<class \'tensorflow.python.keras.mixed_precision.experimental.policy.Policy\'>"
is_instance: "<type \'object\'>"
member {
name: "compute_dtype"
mtype: "<type \'property\'>"
}
member {
name: "loss_scale"
mtype: "<type \'property\'>"
}
member {
name: "name"
mtype: "<type \'property\'>"
}
member {
name: "should_cast_variables"
mtype: "<type \'property\'>"
}
member {
name: "variable_dtype"
mtype: "<type \'property\'>"
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'name\', \'loss_scale\'], varargs=None, keywords=None, defaults=[\'USE_DEFAULT\'], "
}
member_method {
name: "from_config"
argspec: "args=[\'cls\', \'config\', \'custom_objects\'], varargs=None, keywords=None, defaults=[\'None\'], "
}
member_method {
name: "get_config"
argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
}
}

16
tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.pbtxt
View File

@ -4,20 +4,4 @@ tf_module {
name: "LossScaleOptimizer"
mtype: "<type \'type\'>"
}
member {
name: "Policy"
mtype: "<type \'type\'>"
}
member_method {
name: "get_layer_policy"
argspec: "args=[\'layer\'], varargs=None, keywords=None, defaults=None"
}
member_method {
name: "global_policy"
argspec: "args=[], varargs=None, keywords=None, defaults=None"
}
member_method {
name: "set_policy"
argspec: "args=[\'policy\'], varargs=None, keywords=None, defaults=None"
}
}