From f1757f5a586d8cc1b3444485e71bb1c215414909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?= Date: Sun, 14 Oct 2018 16:35:12 +0800 Subject: [PATCH] CLN: remove noisy warning in StepCounterHook --- .../training/basic_session_run_hooks.py | 24 ++++++++----------- .../training/basic_session_run_hooks_test.py | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/tensorflow/python/training/basic_session_run_hooks.py b/tensorflow/python/training/basic_session_run_hooks.py index 1efabcd854d..e7b8b15c47e 100644 --- a/tensorflow/python/training/basic_session_run_hooks.py +++ b/tensorflow/python/training/basic_session_run_hooks.py @@ -653,7 +653,6 @@ class StepCounterHook(session_run_hook.SessionRunHook): self._summary_writer = summary_writer self._output_dir = output_dir self._last_global_step = None - self._global_step_check_count = 0 self._steps_per_run = 1 def _set_steps_per_run(self, steps_per_run): @@ -698,22 +697,19 @@ class StepCounterHook(session_run_hook.SessionRunHook): # step value such that the comparison could be unreliable. For simplicity, # we just compare the stale_global_step with previously recorded version. if stale_global_step == self._last_global_step: - # Here, we use a counter to count how many times we have observed that the + # Here, we give a warning in the first 5 times if we have observed that the # global step has not been increased. For some Optimizers, the global step # is not increased each time by design. For example, SyncReplicaOptimizer # doesn't increase the global step in worker's main train step. - self._global_step_check_count += 1 - if self._global_step_check_count % 20 == 0: - self._global_step_check_count = 0 - logging.warning( - "It seems that global step (tf.train.get_global_step) has not " - "been increased. Current value (could be stable): %s vs previous " - "value: %s. You could increase the global step by passing " - "tf.train.get_global_step() to Optimizer.apply_gradients or " - "Optimizer.minimize.", stale_global_step, self._last_global_step) - else: - # Whenever we observe the increment, reset the counter. - self._global_step_check_count = 0 + logging.log_first_n( + logging.WARN, + "It seems that global step (tf.train.get_global_step) has not " + "been increased. Current value (could be stable): %s vs previous " + "value: %s. You could increase the global step by passing " + "tf.train.get_global_step() to Optimizer.apply_gradients or " + "Optimizer.minimize.", + 5, + stale_global_step, self._last_global_step) self._last_global_step = stale_global_step diff --git a/tensorflow/python/training/basic_session_run_hooks_test.py b/tensorflow/python/training/basic_session_run_hooks_test.py index 2d469634e0e..d3d8a84b36c 100644 --- a/tensorflow/python/training/basic_session_run_hooks_test.py +++ b/tensorflow/python/training/basic_session_run_hooks_test.py @@ -1013,7 +1013,7 @@ class StepCounterHookTest(test.TestCase): hook.begin() mon_sess = monitored_session._HookedSession(sess, [hook]) mon_sess.run(train_op) # Run one step to record global step. - with test.mock.patch.object(tf_logging, 'warning') as mock_log: + with test.mock.patch.object(tf_logging, 'log_first_n') as mock_log: for _ in range(30): mon_sess.run(train_op) self.assertRegexpMatches(