From 09e142227854d758ccfbcb39b60425a818797808 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 12 Aug 2020 16:47:02 +0200 Subject: [PATCH] Reload weights after plateau. --- .gitignore | 1 + training/mozilla_voice_stt_training/train.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 19eaf5e5..eab56fe6 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ Dockerfile.train doc/xml-c doc/xml-java doc/xml-dotnet +convert_graphdef_memmapped_format diff --git a/training/mozilla_voice_stt_training/train.py b/training/mozilla_voice_stt_training/train.py index 7dbfd682..b7f9a66b 100644 --- a/training/mozilla_voice_stt_training/train.py +++ b/training/mozilla_voice_stt_training/train.py @@ -645,6 +645,9 @@ def train(): log_info('Encountered a plateau, reducing learning rate to {}'.format( current_learning_rate)) + # Reload checkpoint that we use the best_dev weights again + load_or_init_graph_for_training(session) + if FLAGS.metrics_files: # Read only metrics, not affecting best validation loss tracking for source, init_op in zip(metrics_sources, metrics_init_ops):