experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

The callback slowness warning has started firing in many situations where only built-in callbacks are called (possibly only due to logging). For the time being, its threshold must be increased.

Browse Source 
PiperOrigin-RevId: 311195250
Change-Id: Idff476f4650970b372bc11a25b043825b17742d5
This commit is contained in:
Francois Chollet 2020-05-12 13:49:58 -07:00 committed by TensorFlower Gardener
parent fd895bf2b9
commit 1186e3f209
2 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tensorflow/python/keras/callbacks.py
View File

@ -307,14 +307,20 @@ class CallbackList(object):
end_hook_name = hook_name
begin_hook_name = 'on_{mode}_batch_begin'.format(mode=mode)
threshold_time = 0.5 * batch_time
threshold_time = 1.5 * batch_time
warning_msg = ('Callbacks method `{hook}` is slow compared to '
'the batch time. Check your callbacks.')
'the batch time (batch time: {batch_time:.4f}s vs '
'`{hook}` time: {cbk_time:.4f}s). Check your callbacks.')
if self._timing[begin_hook_name] > threshold_time:
logging.warning(warning_msg.format(hook=begin_hook_name))
logging.warning(warning_msg.format(
hook=begin_hook_name,
batch_time=batch_time,
cbk_time=self._timing[begin_hook_name]))
if self._timing[end_hook_name] > threshold_time:
logging.warning(warning_msg.format(hook=end_hook_name))
logging.warning(warning_msg.format(
hook=end_hook_name,
batch_time=batch_time,
cbk_time=self._timing[end_hook_name]))
self._check_timing = False
self._batch_start_time = None

4
tensorflow/python/keras/callbacks_test.py
View File

@ -302,8 +302,8 @@ class KerasCallbacksTest(keras_parameterized.TestCase):
epochs=10,
callbacks=[SleepCallback()])
warning_msg = ('Callbacks method `on_train_batch_end` is slow compared '
'to the batch time. Check your callbacks.')
self.assertIn(warning_msg, warning_messages)
'to the batch time')
self.assertIn(warning_msg, '\n'.join(warning_messages))
@keras_parameterized.run_with_all_model_types(exclude_models='functional')
@keras_parameterized.run_all_keras_modes