From 09e142227854d758ccfbcb39b60425a818797808 Mon Sep 17 00:00:00 2001
From: Daniel <daniel@mail.de>
Date: Wed, 12 Aug 2020 16:47:02 +0200
Subject: [PATCH] Reload weights after plateau.

---
 .gitignore                                   | 1 +
 training/mozilla_voice_stt_training/train.py | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index 19eaf5e5..eab56fe6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,4 @@ Dockerfile.train
 doc/xml-c
 doc/xml-java
 doc/xml-dotnet
+convert_graphdef_memmapped_format
diff --git a/training/mozilla_voice_stt_training/train.py b/training/mozilla_voice_stt_training/train.py
index 7dbfd682..b7f9a66b 100644
--- a/training/mozilla_voice_stt_training/train.py
+++ b/training/mozilla_voice_stt_training/train.py
@@ -645,6 +645,9 @@ def train():
                         log_info('Encountered a plateau, reducing learning rate to {}'.format(
                             current_learning_rate))
 
+                        # Reload checkpoint that we use the best_dev weights again
+                        load_or_init_graph_for_training(session)
+
                 if FLAGS.metrics_files:
                     # Read only metrics, not affecting best validation loss tracking
                     for source, init_op in zip(metrics_sources, metrics_init_ops):