From c570cb670ae6f3c262ed325a6d4c1a9125bfed47 Mon Sep 17 00:00:00 2001 From: Yi-Hua Chiu Date: Thu, 2 Jan 2020 11:06:15 +0800 Subject: [PATCH] sparse_warp still can have chance to raise error even after millions steps, so just recover the invertible error while training, unless error raise 3 times, it will be aborted --- DeepSpeech.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/DeepSpeech.py b/DeepSpeech.py index 161ae45b..daa80a8c 100755 --- a/DeepSpeech.py +++ b/DeepSpeech.py @@ -22,6 +22,7 @@ from ds_ctcdecoder import ctc_beam_search_decoder, Scorer from evaluate import evaluate from six.moves import zip, range from tensorflow.python.tools import freeze_graph, strip_unused_lib +from tensorflow.python.framework import errors_impl from util.config import Config, initialize_globals from util.feeding import create_dataset, samples_to_mfccs, audiofile_to_features from util.flags import create_flags, FLAGS @@ -594,9 +595,23 @@ def train(): # Batch loop while True: try: - _, current_step, batch_loss, problem_files, step_summary = \ - session.run([train_op, global_step, loss, non_finite_files, step_summaries_op], - feed_dict=feed_dict) + try: + _, current_step, batch_loss, problem_files, step_summary = \ + session.run([train_op, global_step, loss, non_finite_files, step_summaries_op], + feed_dict=feed_dict) + except errors_impl.InvalidArgumentError as err: + if FLAGS.augmentation_sparse_warp: + # recover twice for sparse warp, if still error, abort it!!! + try: + print('recovering the invertible error: {}'.format(err)) + _, current_step, batch_loss, problem_files, step_summary = \ + session.run([train_op, global_step, loss, non_finite_files, step_summaries_op], + feed_dict=feed_dict) + except errors_impl.InvalidArgumentError as err: + print('recovering the invertible error `AGAIN`: {}'.format(err)) + _, current_step, batch_loss, problem_files, step_summary = \ + session.run([train_op, global_step, loss, non_finite_files, step_summaries_op], + feed_dict=feed_dict) except tf.errors.OutOfRangeError: break