From 2fcba677bbc4e7325e8c4fc7fffe79b9b1cc9582 Mon Sep 17 00:00:00 2001 From: Bernardo Henz Date: Tue, 18 Aug 2020 11:26:39 -0300 Subject: [PATCH 1/5] Implementation of layer-norm in the training script --- training/mozilla_voice_stt_training/train.py | 14 +++++++++----- training/mozilla_voice_stt_training/util/flags.py | 1 + 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/training/mozilla_voice_stt_training/train.py b/training/mozilla_voice_stt_training/train.py index 13c97aa8..8e9c73f7 100644 --- a/training/mozilla_voice_stt_training/train.py +++ b/training/mozilla_voice_stt_training/train.py @@ -74,7 +74,7 @@ def create_overlapping_windows(batch_x): return batch_x -def dense(name, x, units, dropout_rate=None, relu=True): +def dense(name, x, units, dropout_rate=None, relu=True, layer_norm=False): with tfv1.variable_scope(name): bias = variable_on_cpu('bias', [units], tfv1.zeros_initializer()) weights = variable_on_cpu('weights', [x.shape[-1], units], tfv1.keras.initializers.VarianceScaling(scale=1.0, mode="fan_avg", distribution="uniform")) @@ -84,6 +84,10 @@ def dense(name, x, units, dropout_rate=None, relu=True): if relu: output = tf.minimum(tf.nn.relu(output), FLAGS.relu_clip) + if layer_norm: + with tfv1.variable_scope(name): + output = tf.contrib.layers.layer_norm(output) + if dropout_rate is not None: output = tf.nn.dropout(output, rate=dropout_rate) @@ -177,9 +181,9 @@ def create_model(batch_x, seq_length, dropout, reuse=False, batch_size=None, pre # The next three blocks will pass `batch_x` through three hidden layers with # clipped RELU activation and dropout. - layers['layer_1'] = layer_1 = dense('layer_1', batch_x, Config.n_hidden_1, dropout_rate=dropout[0]) - layers['layer_2'] = layer_2 = dense('layer_2', layer_1, Config.n_hidden_2, dropout_rate=dropout[1]) - layers['layer_3'] = layer_3 = dense('layer_3', layer_2, Config.n_hidden_3, dropout_rate=dropout[2]) + layers['layer_1'] = layer_1 = dense('layer_1', batch_x, Config.n_hidden_1, dropout_rate=dropout[0], layer_norm=FLAGS.layer_norm) + layers['layer_2'] = layer_2 = dense('layer_2', layer_1, Config.n_hidden_2, dropout_rate=dropout[1], layer_norm=FLAGS.layer_norm) + layers['layer_3'] = layer_3 = dense('layer_3', layer_2, Config.n_hidden_3, dropout_rate=dropout[2], layer_norm=FLAGS.layer_norm) # `layer_3` is now reshaped into `[n_steps, batch_size, 2*n_cell_dim]`, # as the LSTM RNN expects its input to be of shape `[max_time, batch_size, input_size]`. @@ -196,7 +200,7 @@ def create_model(batch_x, seq_length, dropout, reuse=False, batch_size=None, pre layers['rnn_output_state'] = output_state # Now we feed `output` to the fifth hidden layer with clipped RELU activation - layers['layer_5'] = layer_5 = dense('layer_5', output, Config.n_hidden_5, dropout_rate=dropout[5]) + layers['layer_5'] = layer_5 = dense('layer_5', output, Config.n_hidden_5, dropout_rate=dropout[5], layer_norm=FLAGS.layer_norm) # Now we apply a final linear layer creating `n_classes` dimensional vectors, the logits. layers['layer_6'] = layer_6 = dense('layer_6', layer_5, Config.n_hidden_6, relu=False) diff --git a/training/mozilla_voice_stt_training/util/flags.py b/training/mozilla_voice_stt_training/util/flags.py index e5ad8758..9185d326 100644 --- a/training/mozilla_voice_stt_training/util/flags.py +++ b/training/mozilla_voice_stt_training/util/flags.py @@ -135,6 +135,7 @@ def create_flags(): # Geometry f.DEFINE_integer('n_hidden', 2048, 'layer width to use when initialising layers') + f.DEFINE_boolean('layer_norm', True, 'wether to use layer-normalization after each fully-connected layer (except the last one)') # Initialization From 1f54daf007c4e9c9c1f930dc83add3d9415d8bd2 Mon Sep 17 00:00:00 2001 From: Bernardo Henz Date: Tue, 18 Aug 2020 13:22:01 -0300 Subject: [PATCH 2/5] Default for layer_norm set to False --- training/mozilla_voice_stt_training/util/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/mozilla_voice_stt_training/util/flags.py b/training/mozilla_voice_stt_training/util/flags.py index 9185d326..fe78f0b7 100644 --- a/training/mozilla_voice_stt_training/util/flags.py +++ b/training/mozilla_voice_stt_training/util/flags.py @@ -135,7 +135,7 @@ def create_flags(): # Geometry f.DEFINE_integer('n_hidden', 2048, 'layer width to use when initialising layers') - f.DEFINE_boolean('layer_norm', True, 'wether to use layer-normalization after each fully-connected layer (except the last one)') + f.DEFINE_boolean('layer_norm', False, 'wether to use layer-normalization after each fully-connected layer (except the last one)') # Initialization From b4bc6bfb8a3e2271e0f56f45d95dac12c7595b36 Mon Sep 17 00:00:00 2001 From: Bernardo Henz Date: Mon, 24 Aug 2020 14:23:32 -0300 Subject: [PATCH 3/5] Updating commit of submodule --- tensorflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow b/tensorflow index 4336a5b4..23ad988f 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 4336a5b49fa6d650e24dbdba55bcef9581535244 +Subproject commit 23ad988fcde60fb01f9533e95004bbc4877a9143 From 9f3c40ce48c7854fa7c9de5d48126f153937a015 Mon Sep 17 00:00:00 2001 From: Bernardo Henz Date: Mon, 24 Aug 2020 15:18:06 -0300 Subject: [PATCH 4/5] Replacing old sha with new ones Replacing old sha references ('4336a5b49fa6d650e24dbdba55bcef9581535244') with the new one ('23ad988fcde60fb01f9533e95004bbc4877a9143') --- taskcluster/.shared.yml | 44 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 3749d321..69daba73 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -140,38 +140,38 @@ system: namespace: "project.mozilla-voice-stt.swig.win.amd64.1a4c14945012f1282c2eddc174fb7674d5295de8.0" tensorflow: linux_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.cpu/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.cpu" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu" linux_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.cuda/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda" linux_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.arm/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.arm" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm" linux_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.arm64/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm64/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm64" darwin_amd64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.1.osx/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.1.osx" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.osx/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.osx" android_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.android-arm64/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.android-arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-arm64" android_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.android-armv7/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.android-armv7" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-armv7" win_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.win/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.win" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win" win_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.win-cuda/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.win-cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win-cuda/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win-cuda" ios_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.ios_arm64/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.ios_arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_arm64/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_arm64" ios_x86_64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.ios_x86_64/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.ios_x86_64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64" username: 'build-user' homedir: linux: '/home/build-user' From 8284958f3d8f99dac9ebf65e965e71f5af22ef04 Mon Sep 17 00:00:00 2001 From: Bernardo Henz Date: Mon, 24 Aug 2020 16:19:37 -0300 Subject: [PATCH 5/5] Updating tensorflow version in taskcluster/.build.yml --- taskcluster/.build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 13958a13..40e07d1a 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -26,7 +26,7 @@ build: nc_asset_name: 'native_client.tar.xz' args: tests_cmdline: '' - tensorflow_git_desc: 'TensorFlow: v2.3.0-4-g4336a5b' + tensorflow_git_desc: 'TensorFlow: v2.3.0-6-g23ad988' test_model_task: '' homebrew: url: ''