experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Change default value of loss_scale in Policy.__init__.

Browse Source 
This has no functional effect except the string 'USE_DEFAULT' can longer be passed to the Policy constructor, but I don't think anyone does that. The default value is changed from 'USE_DEFAULT' to 'auto', but the default value as the same effect as before.

This change is described in the RFC: https://github.com/tensorflow/community/pull/293. If an alternative approach ends up being used in the RFC, I will revert this change and implement the other approach.

PiperOrigin-RevId: 336012719
Change-Id: I339a29305276bf1e52af555df9e84090a96db6b8
This commit is contained in:
Reed Wanderman-Milne 2020-10-07 21:24:26 -07:00 committed by TensorFlower Gardener
parent a7b1414b08
commit 37cf5b43c3
2 changed files with 7 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tensorflow/python/keras/mixed_precision/experimental/policy.py
View File

@ -32,11 +32,6 @@ from tensorflow.python.training.experimental import mixed_precision_global_state
from tensorflow.python.util.tf_export import keras_export
# Default value of certain arguments, indicating the default behavior for
# that argument should be used.
USE_DEFAULT = 'USE_DEFAULT'
@keras_export('keras.mixed_precision.experimental.Policy', v1=[])
class Policy(object):
"""A dtype policy for a Keras layer.
@ -293,7 +288,7 @@ class Policy(object):
layer would only work if the inputs were float32.
"""
def __init__(self, name, loss_scale=USE_DEFAULT):
def __init__(self, name, loss_scale='auto'):
"""Constructs the policy.
The `name` argument determines the compute and variable dtype, the default
@ -310,9 +305,10 @@ class Policy(object):
a dynamic loss scale is used. These policies are used for mixed
precision training.
loss_scale: A `tf.mixed_precision.experimental.LossScale`, an int (which
uses a `FixedLossScale`), or the string "dynamic" (which uses a
`DynamicLossScale`). Defaults to using no loss scaling unless `name` is
"mixed_float16", in which case this defaults to "dynamic". Only
uses a `FixedLossScale`), the string "dynamic" (which uses a
`DynamicLossScale`), or None (which uses no loss scale). Defaults to
`"auto"`. In the `"auto"` case: 1) if `name` is `"mixed_float16"`, then
use `loss_scale="dynamic"`. 2) otherwise, do not use a loss scale. Only
`tf.keras.Model`s, not layers, use the loss scale, and it is only used
during `Model.fit`, `Model.train_on_batch`, and other similar methods.
"""
@ -324,7 +320,7 @@ class Policy(object):
self._name = name
self._compute_dtype, self._variable_dtype = self._parse_name(name)
if loss_scale == USE_DEFAULT:
if loss_scale == 'auto':
loss_scale = 'dynamic' if name == 'mixed_float16' else None
self._using_default_loss_scale = True
else:

2
tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-policy.pbtxt
View File

@ -24,7 +24,7 @@ tf_class {
}
member_method {
name: "__init__"
argspec: "args=[\'self\', \'name\', \'loss_scale\'], varargs=None, keywords=None, defaults=[\'USE_DEFAULT\'], "
argspec: "args=[\'self\', \'name\', \'loss_scale\'], varargs=None, keywords=None, defaults=[\'auto\'], "
}
member_method {
name: "from_config"