experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use arithmetic instead of tf.where to set NaN for tf.clip_by_global_norm failure.

Browse Source 
Arithmetic operations are much faster than tf.where.

Also add unit tests for tf.clip_by_global_norm.

PiperOrigin-RevId: 306217896
Change-Id: Ib07da0581550a03fb329480b85ac55349b88ea98
This commit is contained in:
A. Unique TensorFlower 2020-04-13 05:36:25 -07:00 committed by TensorFlower Gardener
parent b7c1d24656
commit b6d4bc7fb6
2 changed files with 56 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tensorflow/python/ops/clip_ops.py
View File

@ -326,11 +326,9 @@ def clip_by_global_norm(t_list, clip_norm, use_norm=None, name=None):
scale_for_finite = clip_norm * math_ops.minimum( scale_for_finite = clip_norm * math_ops.minimum(
1.0 / use_norm, 1.0 / use_norm,
constant_op.constant(1.0, dtype=use_norm.dtype) / clip_norm) constant_op.constant(1.0, dtype=use_norm.dtype) / clip_norm)
scale = array_ops.where( # If use_norm is any finite number, this is a no-op. For inf/-inf/NaN,
math_ops.is_finite(use_norm), # this will make scale NaN.
scale_for_finite, scale = scale_for_finite + (use_norm - use_norm)
# Return NaN if use_norm is not finite.
constant_op.constant(float("nan"), dtype=use_norm.dtype))
values = [ values = [
ops.convert_to_tensor( ops.convert_to_tensor(

38
tensorflow/python/ops/clip_ops_test.py
View File

@ -18,6 +18,8 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
import numpy as np
from tensorflow.python.framework import constant_op from tensorflow.python.framework import constant_op
from tensorflow.python.framework import ops from tensorflow.python.framework import ops
from tensorflow.python.framework import test_util from tensorflow.python.framework import test_util
@ -32,16 +34,24 @@ class ClipOpsTest(test.TestCase):
super(ClipOpsTest, self).__init__(method_name) super(ClipOpsTest, self).__init__(method_name)
def _testClipTensorByNorm(self, inputs, max_norm, expected): def _testClipTensorByNorm(self, inputs, max_norm, expected):
with self.cached_session() as sess:
input_op = constant_op.constant(inputs) input_op = constant_op.constant(inputs)
clipped = clip_ops.clip_by_norm(input_op, max_norm) clipped = clip_ops.clip_by_norm(input_op, max_norm)
check_op = numerics.add_check_numerics_ops() check_op = numerics.add_check_numerics_ops()
result, _ = self.evaluate([clipped, check_op]) result, _ = self.evaluate([clipped, check_op])
self.assertAllClose(result, expected) self.assertAllClose(result, expected)
def _testClipTensorByGlobalNorm(self, inputs, max_norm, expected):
clipped = clip_ops.clip_by_global_norm(inputs, max_norm)
result, _ = self.evaluate(clipped)
self.assertAllClose(result, expected)
def _testNonFiniteClippingByGlobalNorm(self, inputs, max_norm):
clipped = clip_ops.clip_by_global_norm(inputs, max_norm)
result, _ = self.evaluate(clipped)
self.assertTrue(np.all(np.isnan(result)))
def _testClipIndexedSlicesByNorm(self, values, indices, shape, max_norm, def _testClipIndexedSlicesByNorm(self, values, indices, shape, max_norm,
axes): axes):
with self.cached_session() as sess:
values = constant_op.constant(values) values = constant_op.constant(values)
indices = constant_op.constant(indices) indices = constant_op.constant(indices)
shape = constant_op.constant(shape) shape = constant_op.constant(shape)
@ -63,10 +73,34 @@ class ClipOpsTest(test.TestCase):
# Simple example # Simple example
self._testClipTensorByNorm([[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]], 4.0, self._testClipTensorByNorm([[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]], 4.0,
[[-2.4, 0.0, 0.0], [3.2, 0.0, 0.0]]) [[-2.4, 0.0, 0.0], [3.2, 0.0, 0.0]])
# No clipping.
self._testClipTensorByNorm([[1.0, 0.0, 0.0], [1.0, 0.0, 0.0]], 4.0,
[[1.0, 0.0, 0.0], [1.0, 0.0, 0.0]])
# Zero norm # Zero norm
self._testClipTensorByNorm([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], 4.0, self._testClipTensorByNorm([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], 4.0,
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]) [[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])
@test_util.run_deprecated_v1
def testClipTensorByGlobalNorm(self):
# Simple example
self._testClipTensorByGlobalNorm([[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]], 4.0,
[[-2.4, 0.0, 0.0], [3.2, 0.0, 0.0]])
# No clipping.
self._testClipTensorByGlobalNorm([[1.0, 0.0, 0.0], [1.0, 0.0, 0.0]], 4.0,
[[1.0, 0.0, 0.0], [1.0, 0.0, 0.0]])
# Zero norm.
self._testClipTensorByGlobalNorm([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]], 4.0,
[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])
@test_util.run_deprecated_v1
def testGlobalClipWithNonfinite(self):
self._testNonFiniteClippingByGlobalNorm(
[[-3.0, 0.0, 0.0], [float("inf"), 0.0, 0.0]], 4.0)
self._testNonFiniteClippingByGlobalNorm(
[[-3.0, 0.0, 0.0], [float("-inf"), 0.0, 0.0]], 4.0)
self._testNonFiniteClippingByGlobalNorm(
[[-3.0, 0.0, 0.0], [float("nan"), 0.0, 0.0]], 4.0)
def testClipIndexedSlicesByNorm(self): def testClipIndexedSlicesByNorm(self):
values = [[[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]], values = [[[-3.0, 0.0, 0.0], [4.0, 0.0, 0.0]],
[[0.0, 2.0, 0.0], [0.0, 0.0, -1.0]]] [[0.0, 2.0, 0.0], [0.0, 0.0, -1.0]]]