From ed6992c14c9c04a66b26ae345161e7aa8f72d8ae Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 7 Oct 2020 15:57:00 -0700 Subject: [PATCH] Fixed some minor bugs that blocked a learning rate of exactly zero for FTRL. PiperOrigin-RevId: 335970687 Change-Id: I3b2bb3b8c3b567ab7817bd68e72a75fd55278633 --- tensorflow/core/kernels/training_ops.cc | 13 ++++++++----- tensorflow/python/training/ftrl.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/kernels/training_ops.cc b/tensorflow/core/kernels/training_ops.cc index bdb07470c07..252344e796d 100644 --- a/tensorflow/core/kernels/training_ops.cc +++ b/tensorflow/core/kernels/training_ops.cc @@ -2615,11 +2615,14 @@ class SparseApplyFtrlOp : public OpKernel { errors::InvalidArgument("indices must be one-dimensional")); const Tensor& lr = ctx->input(5); - OP_REQUIRES(ctx, - TensorShapeUtils::IsScalar(lr.shape()) && - lr.scalar()() > static_cast(0), - errors::InvalidArgument("lr is not a positive scalar: ", - lr.shape().DebugString())); + OP_REQUIRES( + ctx, + TensorShapeUtils::IsScalar(lr.shape()) && + (lr.scalar()() > static_cast(0) || + (multiply_linear_by_lr_ && lr.scalar()() >= static_cast(0))), + errors::InvalidArgument("lr is not a positive scalar (or zero if " + "multiply_linear_by_lr is set): ", + lr.shape().DebugString())); const Tensor& l1 = ctx->input(6); OP_REQUIRES(ctx, diff --git a/tensorflow/python/training/ftrl.py b/tensorflow/python/training/ftrl.py index 6c8a6ceadc5..21cc0d15ad5 100644 --- a/tensorflow/python/training/ftrl.py +++ b/tensorflow/python/training/ftrl.py @@ -149,7 +149,7 @@ class FtrlOptimizer(optimizer.Optimizer): # TensorFlow ops do not need to include that parameter. self._adjusted_l2_regularization_strength_tensor = ops.convert_to_tensor( self._l2_regularization_strength + self._beta / - (2. * self._learning_rate), + (2. * math_ops.maximum(self._learning_rate, 1e-36)), name="adjusted_l2_regularization_strength") assert self._adjusted_l2_regularization_strength_tensor is not None self._beta_tensor = ops.convert_to_tensor(self._beta, name="beta")