experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Handle NaN results when verifying gradients.

Browse Source 
This change returns NaN as the maximum error if any gradient
entry is NaN. This avoids masking gradient implementation
errors in tests that don't expect to see NaN results.

PiperOrigin-RevId: 207551265
This commit is contained in:
A. Unique TensorFlower 2018-08-06 08:47:03 -07:00 committed by TensorFlower Gardener
parent e01ad771da
commit 291f29b794
2 changed files with 24 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
tensorflow/cc/framework/gradient_checker.cc
View File

@ -351,7 +351,14 @@ Status ComputeGradientErrorInternal(const Scope& scope, const OutputList& xs,
auto jac_n = jacobian_ns[i].matrix<JAC_T>(); auto jac_n = jacobian_ns[i].matrix<JAC_T>();
for (int r = 0; r < jacobian_ts[i].dim_size(0); ++r) { for (int r = 0; r < jacobian_ts[i].dim_size(0); ++r) {
for (int c = 0; c < jacobian_ts[i].dim_size(1); ++c) { for (int c = 0; c < jacobian_ts[i].dim_size(1); ++c) {
*max_error = std::max(*max_error, std::fabs(jac_t(r, c) - jac_n(r, c))); auto cur_error = std::fabs(jac_t(r, c) - jac_n(r, c));
// Treat any NaN as max_error and immediately return.
// (Note that std::max may ignore NaN arguments.)
if (std::isnan(cur_error)) {
*max_error = cur_error;
return Status::OK();
}
*max_error = std::max(*max_error, cur_error);
} }
} }
} }

16
tensorflow/cc/framework/gradient_checker_test.cc
View File

@ -28,12 +28,14 @@ namespace {
using ops::Complex; using ops::Complex;
using ops::Const; using ops::Const;
using ops::Div;
using ops::MatMul; using ops::MatMul;
using ops::Placeholder; using ops::Placeholder;
using ops::Real; using ops::Real;
using ops::Split; using ops::Split;
using ops::Square; using ops::Square;
using ops::Stack; using ops::Stack;
using ops::Sub;
using ops::Unstack; using ops::Unstack;
TEST(GradientCheckerTest, BasicFloat) { TEST(GradientCheckerTest, BasicFloat) {
@ -104,6 +106,20 @@ TEST(GradientCheckerTest, Complex64ToFloat) {
EXPECT_LT(max_error, 1e-4); EXPECT_LT(max_error, 1e-4);
} }
// When calculating gradients that are undefined, test we get NaN
// as the computed error rather than 0.
TEST(GradientCheckerTest, BasicNan) {
Scope scope = Scope::NewRootScope();
TensorShape shape({2, 4, 3});
auto x = Placeholder(scope, DT_FLOAT, Placeholder::Shape(shape));
// y = x/(x-x) should always return NaN
auto y = Div(scope, x, Sub(scope, x, x));
float max_error;
TF_ASSERT_OK((ComputeGradientError<float, float, float>(
scope, {x}, {shape}, {y}, {shape}, &max_error)));
EXPECT_TRUE(std::isnan(max_error));
}
TEST(GradientCheckerTest, MatMulGrad) { TEST(GradientCheckerTest, MatMulGrad) {
Scope scope = Scope::NewRootScope(); Scope scope = Scope::NewRootScope();