From 3353f6a60ae5788fc9397c5d7065bfa272503bbf Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 23 Apr 2020 21:29:31 +0200 Subject: [PATCH 1/2] Update prod model --- taskcluster/test-armbian-opt-base.tyml | 4 ++-- taskcluster/test-darwin-opt-base.tyml | 4 ++-- taskcluster/test-linux-opt-base.tyml | 4 ++-- taskcluster/test-raspbian-opt-base.tyml | 4 ++-- taskcluster/test-win-cuda-opt-base.tyml | 4 ++-- taskcluster/test-win-opt-base.tyml | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/taskcluster/test-armbian-opt-base.tyml b/taskcluster/test-armbian-opt-base.tyml index e6d1f28a..7218e684 100644 --- a/taskcluster/test-armbian-opt-base.tyml +++ b/taskcluster/test-armbian-opt-base.tyml @@ -28,8 +28,8 @@ then: in: DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_arm64_build}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb - DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pb - DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pbmm + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://lissyx.github.io/deepspeech-python-wheels/" EXTRA_PYTHON_CONFIGURE_OPTS: "" # Required by Debian Buster diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index a11f3d8f..d35aef89 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -33,8 +33,8 @@ then: DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${darwin_amd64_build}/artifacts/public DEEPSPEECH_ARTIFACTS_TFLITE_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${darwin_amd64_tflite}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb - DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pb - DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pbmm + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm EXPECTED_TENSORFLOW_VERSION: "${build.tensorflow_git_desc}" command: diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index a8cde72b..fcb8b466 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -31,8 +31,8 @@ then: DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_build}/artifacts/public DEEPSPEECH_ARTIFACTS_TFLITE_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_tflite}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb - DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pb - DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pbmm + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm DECODER_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_amd64_ctc}/artifacts/public PIP_DEFAULT_TIMEOUT: "60" EXPECTED_TENSORFLOW_VERSION: "${build.tensorflow_git_desc}" diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index 07fc54a8..f3d2a5e5 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -28,8 +28,8 @@ then: in: DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${linux_rpi3_build}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb - DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pb - DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pbmm + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm PIP_DEFAULT_TIMEOUT: "60" PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple" EXTRA_PYTHON_CONFIGURE_OPTS: "" # Required by Raspbian Buster / PiWheels diff --git a/taskcluster/test-win-cuda-opt-base.tyml b/taskcluster/test-win-cuda-opt-base.tyml index c5bedf30..0549fa8e 100644 --- a/taskcluster/test-win-cuda-opt-base.tyml +++ b/taskcluster/test-win-cuda-opt-base.tyml @@ -27,8 +27,8 @@ then: in: DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${win_amd64_build}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb - DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pb - DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pbmm + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm EXPECTED_TENSORFLOW_VERSION: "${build.tensorflow_git_desc}" TC_MSYS_VERSION: 'MSYS_NT-6.3' MSYS: 'winsymlinks:nativestrict' diff --git a/taskcluster/test-win-opt-base.tyml b/taskcluster/test-win-opt-base.tyml index 72a52702..e95a3037 100644 --- a/taskcluster/test-win-opt-base.tyml +++ b/taskcluster/test-win-opt-base.tyml @@ -29,8 +29,8 @@ then: DEEPSPEECH_ARTIFACTS_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${win_amd64_build}/artifacts/public DEEPSPEECH_ARTIFACTS_TFLITE_ROOT: https://community-tc.services.mozilla.com/api/queue/v1/task/${win_amd64_tflite}/artifacts/public DEEPSPEECH_TEST_MODEL: https://community-tc.services.mozilla.com/api/queue/v1/task/${training}/artifacts/public/output_graph.pb - DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pb - DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.6.0-alpha.15/output_graph.pbmm + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm EXPECTED_TENSORFLOW_VERSION: "${build.tensorflow_git_desc}" TC_MSYS_VERSION: 'MSYS_NT-6.3' MSYS: 'winsymlinks:nativestrict' From ff8570c1f3c2182b98c9284a9870e42ca455a2bc Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 23 Apr 2020 23:32:07 +0200 Subject: [PATCH 2/2] Update prod model expected inference results --- taskcluster/tc-asserts.sh | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/taskcluster/tc-asserts.sh b/taskcluster/tc-asserts.sh index b12b9683..0f70e0c2 100755 --- a/taskcluster/tc-asserts.sh +++ b/taskcluster/tc-asserts.sh @@ -196,45 +196,33 @@ assert_correct_multi_ldc93s1() assert_correct_ldc93s1_prodmodel() { if [ -z "$3" -o "$3" = "16k" ]; then - assert_correct_inference "$1" "she had reduce and greasy wash water all year" "$2" + assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" "$2" fi; if [ "$3" = "8k" ]; then - assert_correct_inference "$1" "she had conduct suit in greasy wash water all year" "$2" + assert_correct_inference "$1" "she had to do suit in greasy wash water all year" "$2" fi; } assert_correct_ldc93s1_prodtflitemodel() { if [ -z "$3" -o "$3" = "16k" ]; then - assert_correct_inference "$1" "she had i do utterly was or all year" "$2" + assert_correct_inference "$1" "she had her dark suit in greasy wash water all year" "$2" fi; if [ "$3" = "8k" ]; then - assert_correct_inference "$1" "she had up a out and we wash or a" "$2" + assert_correct_inference "$1" "she had to do so in greasy wash water all year" "$2" fi; } assert_correct_ldc93s1_prodmodel_stereo_44k() { - if [ -z "$3" -o "$3" = "16k" ]; then - assert_correct_inference "$1" "she had reduce and greasy wash water all year" "$2" - fi; - - if [ "$3" = "8k" ]; then - assert_correct_inference "$1" "she had reduce and greasy wash water all year" "$2" - fi; + assert_correct_inference "$1" "she had your dark suit in greasy wash water all year" "$2" } assert_correct_ldc93s1_prodtflitemodel_stereo_44k() { - if [ -z "$3" -o "$3" = "16k" ]; then - assert_correct_inference "$1" "she headed grey was or all year" "$2" - fi; - - if [ "$3" = "8k" ]; then - assert_correct_inference "$1" "she headed grey was or all year" "$2" - fi; + assert_correct_inference "$1" "she had her dark suit in greasy wash water all year" "$2" } assert_correct_warning_upsampling() @@ -458,7 +446,7 @@ run_prod_inference_tests() phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) status=$? set -e - assert_correct_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status" "${_bitrate}" + assert_correct_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status" # Run down-sampling warning test only when we actually perform downsampling if [ "${ldc93s1_sample_filename}" != "LDC93S1_pcms16le_1_8000.wav" ]; then @@ -489,7 +477,7 @@ run_prodtflite_inference_tests() phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) status=$? set -e - assert_correct_ldc93s1_prodtflitemodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status" "${_bitrate}" + assert_correct_ldc93s1_prodtflitemodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status" # Run down-sampling warning test only when we actually perform downsampling if [ "${ldc93s1_sample_filename}" != "LDC93S1_pcms16le_1_8000.wav" ]; then