diff --git a/.github/workflows/macOS-amd64.yml b/.github/workflows/macOS-amd64.yml index eabd742d..f7ad40f9 100644 --- a/.github/workflows/macOS-amd64.yml +++ b/.github/workflows/macOS-amd64.yml @@ -5,8 +5,8 @@ on: branches: - master env: - TASKCLUSTER_TASK_DIR: ${{ github.workspace }} - TASKCLUSTER_ARTIFACTS: ${{ github.workspace }}/artifacts + CI_TASK_DIR: ${{ github.workspace }} + CI_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts MACOSX_DEPLOYMENT_TARGET: "10.10" jobs: swig_macOS: @@ -357,7 +357,7 @@ jobs: models: ["test", "prod"] bitrate: ["8k", "16k"] env: - TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/ + CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb @@ -369,17 +369,17 @@ jobs: - uses: actions/download-artifact@v2 with: name: "native_client.${{ matrix.build-flavor }}.tar.xz" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} - run: | - cd ${{ env.TASKCLUSTER_TMP_DIR }} + cd ${{ env.CI_TMP_DIR }} mkdir ds && cd ds && tar xf ../native_client.tar.xz - uses: actions/download-artifact@v2 with: name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + ls -hal ${{ env.CI_TMP_DIR }}/ if: matrix.models == 'test' - uses: ./.github/actions/run-tests with: @@ -399,7 +399,7 @@ jobs: models: ["test", "prod"] bitrate: ["8k", "16k"] env: - TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/ + CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb @@ -414,18 +414,18 @@ jobs: - uses: actions/download-artifact@v2 with: name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} - uses: actions/download-artifact@v2 with: name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + ls -hal ${{ env.CI_TMP_DIR }}/ if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ - pip3 install --only-binary :all: --upgrade ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.whl + ls -hal ${{ env.CI_TMP_DIR }}/ + pip3 install --only-binary :all: --upgrade ${{ env.CI_TMP_DIR }}/stt*.whl - uses: ./.github/actions/run-tests with: runtime: "python" @@ -474,7 +474,7 @@ jobs: models: ["test"] bitrate: ["16k"] env: - TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/ + CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb @@ -489,18 +489,18 @@ jobs: - uses: actions/download-artifact@v2 with: name: "stt_intermediate-${{ matrix.build-flavor }}.tgz" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} - uses: actions/download-artifact@v2 with: name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + ls -hal ${{ env.CI_TMP_DIR }}/ if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ - npm install ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.tgz + ls -hal ${{ env.CI_TMP_DIR }}/ + npm install ${{ env.CI_TMP_DIR }}/stt*.tgz - uses: ./.github/actions/run-tests with: runtime: "node" @@ -519,7 +519,7 @@ jobs: models: ["test"] bitrate: ["16k"] env: - TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/ + CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb @@ -534,18 +534,18 @@ jobs: - uses: actions/download-artifact@v2 with: name: "stt_intermediate-${{ matrix.build-flavor }}.tgz" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} - uses: actions/download-artifact@v2 with: name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + ls -hal ${{ env.CI_TMP_DIR }}/ if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ - npm install ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.tgz + ls -hal ${{ env.CI_TMP_DIR }}/ + npm install ${{ env.CI_TMP_DIR }}/stt*.tgz - run: | npm install electron@${{ matrix.electronjs-version }} - uses: ./.github/actions/run-tests @@ -568,7 +568,7 @@ jobs: models: ["test", "prod"] bitrate: ["8k", "16k"] env: - TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/ + CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb @@ -583,18 +583,18 @@ jobs: - uses: actions/download-artifact@v2 with: name: "stt-${{ matrix.build-flavor }}.tgz" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} - uses: actions/download-artifact@v2 with: name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + ls -hal ${{ env.CI_TMP_DIR }}/ if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ - npm install ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.tgz + ls -hal ${{ env.CI_TMP_DIR }}/ + npm install ${{ env.CI_TMP_DIR }}/stt*.tgz - uses: ./.github/actions/run-tests with: runtime: "node" @@ -613,7 +613,7 @@ jobs: models: ["test", "prod"] bitrate: ["8k", "16k"] env: - TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/ + CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb @@ -628,18 +628,18 @@ jobs: - uses: actions/download-artifact@v2 with: name: "stt-${{ matrix.build-flavor }}.tgz" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} - uses: actions/download-artifact@v2 with: name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" - path: ${{ env.TASKCLUSTER_TMP_DIR }} + path: ${{ env.CI_TMP_DIR }} if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + ls -hal ${{ env.CI_TMP_DIR }}/ if: matrix.models == 'test' - run: | - ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ - npm install ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.tgz + ls -hal ${{ env.CI_TMP_DIR }}/ + npm install ${{ env.CI_TMP_DIR }}/stt*.tgz - run: | npm install electron@${{ matrix.electronjs-version }} - uses: ./.github/actions/run-tests diff --git a/ci_scripts/all-utils.sh b/ci_scripts/all-utils.sh index 6df7def8..e73eecf4 100755 --- a/ci_scripts/all-utils.sh +++ b/ci_scripts/all-utils.sh @@ -24,26 +24,26 @@ set_ldc_sample_filename() download_model_prod() { local _model_source_file=$(basename "${model_source}") - ${WGET} "${model_source}" -O - | gunzip --force > "${TASKCLUSTER_TMP_DIR}/${_model_source_file}" + ${WGET} "${model_source}" -O - | gunzip --force > "${CI_TMP_DIR}/${_model_source_file}" local _model_source_mmap_file=$(basename "${model_source_mmap}") - ${WGET} "${model_source_mmap}" -O - | gunzip --force > "${TASKCLUSTER_TMP_DIR}/${_model_source_mmap_file}" + ${WGET} "${model_source_mmap}" -O - | gunzip --force > "${CI_TMP_DIR}/${_model_source_mmap_file}" } download_data() { - cp ${DS_DSDIR}/data/smoke_test/*.wav ${TASKCLUSTER_TMP_DIR}/ - cp ${DS_DSDIR}/data/smoke_test/pruned_lm.scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer - cp ${DS_DSDIR}/data/smoke_test/pruned_lm.bytes.scorer ${TASKCLUSTER_TMP_DIR}/kenlm.bytes.scorer + cp ${DS_DSDIR}/data/smoke_test/*.wav ${CI_TMP_DIR}/ + cp ${DS_DSDIR}/data/smoke_test/pruned_lm.scorer ${CI_TMP_DIR}/kenlm.scorer + cp ${DS_DSDIR}/data/smoke_test/pruned_lm.bytes.scorer ${CI_TMP_DIR}/kenlm.bytes.scorer - cp -R ${DS_DSDIR}/native_client/test ${TASKCLUSTER_TMP_DIR}/test_sources + cp -R ${DS_DSDIR}/native_client/test ${CI_TMP_DIR}/test_sources } download_material() { download_data - ls -hal ${TASKCLUSTER_TMP_DIR}/${model_name} ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} ${TASKCLUSTER_TMP_DIR}/LDC93S1*.wav + ls -hal ${CI_TMP_DIR}/${model_name} ${CI_TMP_DIR}/${model_name_mmap} ${CI_TMP_DIR}/LDC93S1*.wav } maybe_install_xldd() @@ -81,9 +81,9 @@ verify_bazel_rebuild() exit 1 fi; - mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + mkdir -p ${CI_ARTIFACTS_DIR} || true - cp ${DS_DSDIR}/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ + cp ${DS_DSDIR}/tensorflow/bazel*.log ${CI_ARTIFACTS_DIR}/ spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|[for host]|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libstt.so' | wc -l) if [ "${spurious_rebuilds}" -ne 0 ]; then @@ -95,8 +95,8 @@ verify_bazel_rebuild() tar xf ${DS_ROOT_TASK}/bazel-ckd-ds.tar --strip-components=4 -C ${DS_DSDIR}/ckd/tensorflow/ echo "Making a diff between CKD files" - mkdir -p ${TASKCLUSTER_ARTIFACTS} - diff -urNw ${DS_DSDIR}/ckd/tensorflow/ ${DS_ROOT_TASK}/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff + mkdir -p ${CI_ARTIFACTS_DIR} + diff -urNw ${DS_DSDIR}/ckd/tensorflow/ ${DS_ROOT_TASK}/ckd/ds/ | tee ${CI_ARTIFACTS_DIR}/ckd.diff rm -fr ${DS_DSDIR}/ckd/tensorflow/ ${DS_ROOT_TASK}/ckd/ds/ else diff --git a/ci_scripts/all-vars.sh b/ci_scripts/all-vars.sh index 3b7d9d3d..93db8c6b 100755 --- a/ci_scripts/all-vars.sh +++ b/ci_scripts/all-vars.sh @@ -9,10 +9,10 @@ if [ "${OS}" = "Linux" ]; then export DS_CPU_COUNT=$(nproc) fi; -if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then - export TASKCLUSTER_TASK_DIR="$(cygpath ${TASKCLUSTER_TASK_DIR})" - export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} - export PYENV_ROOT="${TASKCLUSTER_TASK_DIR}/pyenv-root" +if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then + export CI_TASK_DIR="$(cygpath ${CI_TASK_DIR})" + export DS_ROOT_TASK=${CI_TASK_DIR} + export PYENV_ROOT="${CI_TASK_DIR}/pyenv-root" export PLATFORM_EXE_SUFFIX=.exe export DS_CPU_COUNT=$(nproc) @@ -21,15 +21,15 @@ if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then fi; if [ "${OS}" = "Darwin" ]; then - export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} + export DS_ROOT_TASK=${CI_TASK_DIR} export DS_CPU_COUNT=$(sysctl hw.ncpu |cut -d' ' -f2) export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root" export HOMEBREW_NO_AUTO_UPDATE=1 export BREW_URL=https://github.com/Homebrew/brew/tarball/2.2.17 - export BUILDS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-builds" - export TESTS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-tests" + export BUILDS_BREW="${CI_TASK_DIR}/homebrew-builds" + export TESTS_BREW="${CI_TASK_DIR}/homebrew-tests" export NVM_DIR=$TESTS_BREW/.nvm/ && mkdir -p $NVM_DIR export PKG_CONFIG_PATH="${BUILDS_BREW}/lib/pkgconfig" @@ -43,12 +43,12 @@ if [ "${OS}" = "Darwin" ]; then fi; fi; -export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts} -export TASKCLUSTER_TMP_DIR=${TASKCLUSTER_TMP_DIR:-/tmp} +export CI_ARTIFACTS_DIR=${CI_ARTIFACTS_DIR:-/tmp/artifacts} +export CI_TMP_DIR=${CI_TMP_DIR:-/tmp} export ANDROID_TMP_DIR=/data/local/tmp -mkdir -p ${TASKCLUSTER_TMP_DIR} || true +mkdir -p ${CI_TMP_DIR} || true export DS_TFDIR=${DS_ROOT_TASK}/tensorflow export DS_DSDIR=${DS_ROOT_TASK}/ @@ -71,7 +71,7 @@ if [ "${OS}" = "Darwin" ]; then TAR="gtar" fi -if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then +if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then WGET=/usr/bin/wget.exe TAR=/usr/bin/tar.exe XZ="xz -9 -T0 -c -" diff --git a/ci_scripts/asserts.sh b/ci_scripts/asserts.sh index e75e662d..9343c663 100755 --- a/ci_scripts/asserts.sh +++ b/ci_scripts/asserts.sh @@ -14,7 +14,7 @@ assert_correct_inference() status=$3 if [ "$status" -ne "0" ]; then - case "$(cat ${TASKCLUSTER_TMP_DIR}/stderr)" in + case "$(cat ${CI_TMP_DIR}/stderr)" in *"incompatible with minimum version"*) echo "Prod model too old for client, skipping test." return 0 @@ -22,7 +22,7 @@ assert_correct_inference() *) echo "Client failed to run:" - cat ${TASKCLUSTER_TMP_DIR}/stderr + cat ${CI_TMP_DIR}/stderr return 1 ;; esac @@ -69,7 +69,7 @@ assert_working_inference() fi; if [ "$status" -ne "0" ]; then - case "$(cat ${TASKCLUSTER_TMP_DIR}/stderr)" in + case "$(cat ${CI_TMP_DIR}/stderr)" in *"incompatible with minimum version"*) echo "Prod model too old for client, skipping test." return 0 @@ -77,7 +77,7 @@ assert_working_inference() *) echo "Client failed to run:" - cat ${TASKCLUSTER_TMP_DIR}/stderr + cat ${CI_TMP_DIR}/stderr return 1 ;; esac @@ -266,7 +266,7 @@ ensure_cuda_usage() if [ "${_maybe_cuda}" = "cuda" ]; then set +e export TF_CPP_MIN_VLOG_LEVEL=1 - ds_cuda=$(${DS_BINARY_PREFIX}${DS_BINARY_FILE} --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>&1 1>/dev/null) + ds_cuda=$(${DS_BINARY_PREFIX}${DS_BINARY_FILE} --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>&1 1>/dev/null) export TF_CPP_MIN_VLOG_LEVEL= set -e @@ -278,7 +278,7 @@ ensure_cuda_usage() check_versions() { set +e - ds_help=$(${DS_BINARY_PREFIX}stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>&1 1>/dev/null) + ds_help=$(${DS_BINARY_PREFIX}stt --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>&1 1>/dev/null) set -e assert_tensorflow_version "${ds_help}" @@ -309,12 +309,12 @@ check_runtime_electronjs() run_tflite_basic_inference_tests() { set +e - phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) set -e assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$?" set +e - phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${CI_TMP_DIR}/stderr) set -e assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$?" } @@ -322,22 +322,22 @@ run_tflite_basic_inference_tests() run_netframework_inference_tests() { set +e - phrase_pbmodel_nolm=$(STTConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(STTConsole.exe --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) set -e assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?" set +e - phrase_pbmodel_nolm=$(STTConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended yes 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(STTConsole.exe --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --extended yes 2>${CI_TMP_DIR}/stderr) set -e assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?" set +e - phrase_pbmodel_nolm=$(STTConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(STTConsole.exe --model ${CI_TMP_DIR}/${model_name_mmap} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) set -e assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?" set +e - phrase_pbmodel_withlm=$(STTConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm=$(STTConsole.exe --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) set -e assert_working_ldc93s1_lm "${phrase_pbmodel_withlm}" "$?" } @@ -345,22 +345,22 @@ run_netframework_inference_tests() run_electronjs_inference_tests() { set +e - phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(stt --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) set -e assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?" set +e - phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(stt --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${CI_TMP_DIR}/stderr) set -e assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?" set +e - phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) set -e assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?" set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) set -e assert_working_ldc93s1_lm "${phrase_pbmodel_withlm}" "$?" } @@ -368,30 +368,30 @@ run_electronjs_inference_tests() run_basic_inference_tests() { set +e - stt --model "" --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr + stt --model "" --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr set -e - grep "Missing model information" ${TASKCLUSTER_TMP_DIR}/stderr + grep "Missing model information" ${CI_TMP_DIR}/stderr set +e - phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(stt --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$status" set +e - phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(stt --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$status" set +e - phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$status" set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" "$status" @@ -402,13 +402,13 @@ run_all_inference_tests() run_basic_inference_tests set +e - phrase_pbmodel_nolm_stereo_44k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_nolm_stereo_44k=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}" "$status" set +e - phrase_pbmodel_withlm_stereo_44k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm_stereo_44k=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm_stereo_44k}" "$status" @@ -416,12 +416,12 @@ run_all_inference_tests() # Run down-sampling warning test only when we actually perform downsampling if [ "${ldc93s1_sample_filename}" != "LDC93S1_pcms16le_1_8000.wav" ]; then set +e - phrase_pbmodel_nolm_mono_8k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) + phrase_pbmodel_nolm_mono_8k=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) set -e assert_correct_warning_upsampling "${phrase_pbmodel_nolm_mono_8k}" set +e - phrase_pbmodel_withlm_mono_8k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) + phrase_pbmodel_withlm_mono_8k=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) set -e assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}" fi; @@ -432,11 +432,11 @@ run_prod_concurrent_stream_tests() local _bitrate=$1 set +e - output=$(python3 ${TASKCLUSTER_TMP_DIR}/test_sources/concurrent_streams.py \ - --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} \ - --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer \ - --audio1 ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_16000.wav \ - --audio2 ${TASKCLUSTER_TMP_DIR}/new-home-in-the-stars-16k.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + output=$(python3 ${CI_TMP_DIR}/test_sources/concurrent_streams.py \ + --model ${CI_TMP_DIR}/${model_name_mmap} \ + --scorer ${CI_TMP_DIR}/kenlm.scorer \ + --audio1 ${CI_TMP_DIR}/LDC93S1_pcms16le_1_16000.wav \ + --audio2 ${CI_TMP_DIR}/new-home-in-the-stars-16k.wav 2>${CI_TMP_DIR}/stderr) status=$? set -e @@ -452,19 +452,19 @@ run_prod_inference_tests() local _bitrate=$1 set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}" set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}" set +e - phrase_pbmodel_withlm_stereo_44k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm_stereo_44k=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status" @@ -472,7 +472,7 @@ run_prod_inference_tests() # Run down-sampling warning test only when we actually perform downsampling if [ "${ldc93s1_sample_filename}" != "LDC93S1_pcms16le_1_8000.wav" ]; then set +e - phrase_pbmodel_withlm_mono_8k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) + phrase_pbmodel_withlm_mono_8k=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) set -e assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}" fi; @@ -483,19 +483,19 @@ run_prodtflite_inference_tests() local _bitrate=$1 set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}" set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}" set +e - phrase_pbmodel_withlm_stereo_44k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr) + phrase_pbmodel_withlm_stereo_44k=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_prodtflitemodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status" @@ -503,7 +503,7 @@ run_prodtflite_inference_tests() # Run down-sampling warning test only when we actually perform downsampling if [ "${ldc93s1_sample_filename}" != "LDC93S1_pcms16le_1_8000.wav" ]; then set +e - phrase_pbmodel_withlm_mono_8k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) + phrase_pbmodel_withlm_mono_8k=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null) set -e assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}" fi; @@ -512,13 +512,13 @@ run_prodtflite_inference_tests() run_multi_inference_tests() { set +e -o pipefail - multi_phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/ 2>${TASKCLUSTER_TMP_DIR}/stderr | tr '\n' '%') + multi_phrase_pbmodel_nolm=$(stt --model ${CI_TMP_DIR}/${model_name} --audio ${CI_TMP_DIR}/ 2>${CI_TMP_DIR}/stderr | tr '\n' '%') status=$? set -e +o pipefail assert_correct_multi_ldc93s1 "${multi_phrase_pbmodel_nolm}" "$status" set +e -o pipefail - multi_phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/ 2>${TASKCLUSTER_TMP_DIR}/stderr | tr '\n' '%') + multi_phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/ 2>${CI_TMP_DIR}/stderr | tr '\n' '%') status=$? set -e +o pipefail assert_correct_multi_ldc93s1 "${multi_phrase_pbmodel_withlm}" "$status" @@ -528,7 +528,7 @@ run_hotword_tests() { DS_BINARY_FILE=${DS_BINARY_FILE:-"stt"} set +e - hotwords_decode=$(${DS_BINARY_PREFIX}${DS_BINARY_FILE} --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --hot_words "foo:0.0,bar:-0.1" 2>${TASKCLUSTER_TMP_DIR}/stderr) + hotwords_decode=$(${DS_BINARY_PREFIX}${DS_BINARY_FILE} --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --hot_words "foo:0.0,bar:-0.1" 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_working_ldc93s1_lm "${hotwords_decode}" "$status" @@ -537,7 +537,7 @@ run_hotword_tests() run_android_hotword_tests() { set +e - hotwords_decode=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --scorer ${DATA_TMP_DIR}/kenlm.scorer --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} --hot_words "foo:0.0,bar:-0.1" 2>${TASKCLUSTER_TMP_DIR}/stderr) + hotwords_decode=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --scorer ${DATA_TMP_DIR}/kenlm.scorer --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} --hot_words "foo:0.0,bar:-0.1" 2>${CI_TMP_DIR}/stderr) status=$? set -e assert_correct_ldc93s1_lm "${hotwords_decode}" "$status" @@ -546,7 +546,7 @@ run_android_hotword_tests() run_cpp_only_inference_tests() { set +e - phrase_pbmodel_withlm_intermediate_decode=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 1280 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1) + phrase_pbmodel_withlm_intermediate_decode=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --stream 1280 2>${CI_TMP_DIR}/stderr | tail -n 1) status=$? set -e assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm_intermediate_decode}" "$status" @@ -555,13 +555,13 @@ run_cpp_only_inference_tests() run_js_streaming_inference_tests() { set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${CI_TMP_DIR}/stderr | tail -n 1) status=$? set -e assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" "$status" set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${CI_TMP_DIR}/stderr | tail -n 1) status=$? set -e assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" "$status" @@ -571,14 +571,14 @@ run_js_streaming_prod_inference_tests() { local _bitrate=$1 set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${CI_TMP_DIR}/stderr | tail -n 1) status=$? set -e assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}" local _bitrate=$1 set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${CI_TMP_DIR}/stderr | tail -n 1) status=$? set -e assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}" @@ -588,14 +588,14 @@ run_js_streaming_prodtflite_inference_tests() { local _bitrate=$1 set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${CI_TMP_DIR}/stderr | tail -n 1) status=$? set -e assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}" local _bitrate=$1 set +e - phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1) + phrase_pbmodel_withlm=$(stt --model ${CI_TMP_DIR}/${model_name_mmap} --scorer ${CI_TMP_DIR}/kenlm.scorer --audio ${CI_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${CI_TMP_DIR}/stderr | tail -n 1) status=$? set -e assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}" diff --git a/ci_scripts/cpp-bytes-tests.sh b/ci_scripts/cpp-bytes-tests.sh index a1c544da..4d762c64 100755 --- a/ci_scripts/cpp-bytes-tests.sh +++ b/ci_scripts/cpp-bytes-tests.sh @@ -9,9 +9,9 @@ source $(dirname "$0")/asserts.sh bitrate=$1 set_ldc_sample_filename "${bitrate}" -download_material "${TASKCLUSTER_TMP_DIR}/ds" +download_material "${CI_TMP_DIR}/ds" -export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +export PATH=${CI_TMP_DIR}/ds/:$PATH # Bytes output mode with LDC93S1 takes too long to converge so we simply test # that loading the model won't crash diff --git a/ci_scripts/cpp-tests-prod.sh b/ci_scripts/cpp-tests-prod.sh index a0c0eb7a..79c1d994 100755 --- a/ci_scripts/cpp-tests-prod.sh +++ b/ci_scripts/cpp-tests-prod.sh @@ -19,7 +19,7 @@ download_model_prod download_material -export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +export PATH=${CI_TMP_DIR}/ds/:$PATH check_versions diff --git a/ci_scripts/cpp-tests.sh b/ci_scripts/cpp-tests.sh index 5d010f0d..460fc36d 100755 --- a/ci_scripts/cpp-tests.sh +++ b/ci_scripts/cpp-tests.sh @@ -11,7 +11,7 @@ set_ldc_sample_filename "${bitrate}" download_data -export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +export PATH=${CI_TMP_DIR}/ds/:$PATH check_versions diff --git a/ci_scripts/cpp_tflite-tests-prod.sh b/ci_scripts/cpp_tflite-tests-prod.sh index 3a4ab453..c9a231d3 100755 --- a/ci_scripts/cpp_tflite-tests-prod.sh +++ b/ci_scripts/cpp_tflite-tests-prod.sh @@ -13,13 +13,13 @@ model_source=${STT_PROD_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") model_name_mmap=$(basename "${model_source}") model_source_mmap=${STT_PROD_MODEL_MMAP//.pbmm/.tflite} -export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR} +export DATA_TMP_DIR=${CI_TMP_DIR} download_model_prod -download_material "${TASKCLUSTER_TMP_DIR}/ds" +download_material "${CI_TMP_DIR}/ds" -export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +export PATH=${CI_TMP_DIR}/ds/:$PATH check_versions diff --git a/ci_scripts/cpp_tflite-tests.sh b/ci_scripts/cpp_tflite-tests.sh index b8733b44..f5c7ebb4 100755 --- a/ci_scripts/cpp_tflite-tests.sh +++ b/ci_scripts/cpp_tflite-tests.sh @@ -12,11 +12,11 @@ set_ldc_sample_filename "${bitrate}" model_source=${STT_TEST_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") model_name_mmap=$(basename "${model_source}") -export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR} +export DATA_TMP_DIR=${CI_TMP_DIR} -download_material "${TASKCLUSTER_TMP_DIR}/ds" +download_material "${CI_TMP_DIR}/ds" -export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +export PATH=${CI_TMP_DIR}/ds/:$PATH check_versions diff --git a/ci_scripts/cpp_tflite_basic-tests.sh b/ci_scripts/cpp_tflite_basic-tests.sh index fe7947c0..35099aec 100755 --- a/ci_scripts/cpp_tflite_basic-tests.sh +++ b/ci_scripts/cpp_tflite_basic-tests.sh @@ -11,11 +11,11 @@ set_ldc_sample_filename "${bitrate}" model_source=${STT_TEST_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") -export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR} +export DATA_TMP_DIR=${CI_TMP_DIR} -download_material "${TASKCLUSTER_TMP_DIR}/ds" +download_material "${CI_TMP_DIR}/ds" -export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +export PATH=${CI_TMP_DIR}/ds/:$PATH check_versions diff --git a/ci_scripts/cppwin-tests.sh b/ci_scripts/cppwin-tests.sh index 4cafc6ba..81acf862 100755 --- a/ci_scripts/cppwin-tests.sh +++ b/ci_scripts/cppwin-tests.sh @@ -9,9 +9,9 @@ source $(dirname "$0")/asserts.sh bitrate=$1 set_ldc_sample_filename "${bitrate}" -download_material "${TASKCLUSTER_TMP_DIR}/ds" +download_material "${CI_TMP_DIR}/ds" -export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +export PATH=${CI_TMP_DIR}/ds/:$PATH check_versions diff --git a/ci_scripts/package-utils.sh b/ci_scripts/package-utils.sh index 6193a1a1..f0162dd0 100755 --- a/ci_scripts/package-utils.sh +++ b/ci_scripts/package-utils.sh @@ -6,7 +6,7 @@ package_native_client() { tensorflow_dir=${DS_TFDIR} stt_dir=${DS_DSDIR} - artifacts_dir=${TASKCLUSTER_ARTIFACTS} + artifacts_dir=${CI_ARTIFACTS_DIR} artifact_name=$1 if [ ! -d ${tensorflow_dir} -o ! -d ${stt_dir} -o ! -d ${artifacts_dir} ]; then @@ -41,7 +41,7 @@ package_native_client_ndk() { stt_dir=${DS_DSDIR} tensorflow_dir=${DS_TFDIR} - artifacts_dir=${TASKCLUSTER_ARTIFACTS} + artifacts_dir=${CI_ARTIFACTS_DIR} artifact_name=$1 arch_abi=$2 @@ -74,7 +74,7 @@ package_native_client_ndk() package_libstt_as_zip() { tensorflow_dir=${DS_TFDIR} - artifacts_dir=${TASKCLUSTER_ARTIFACTS} + artifacts_dir=${CI_ARTIFACTS_DIR} artifact_name=$1 if [ ! -d ${tensorflow_dir} -o ! -d ${artifacts_dir} ]; then diff --git a/ci_scripts/package.sh b/ci_scripts/package.sh index a0401d84..055ac0aa 100755 --- a/ci_scripts/package.sh +++ b/ci_scripts/package.sh @@ -5,19 +5,19 @@ set -xe source $(dirname "$0")/all-vars.sh source $(dirname "$0")/package-utils.sh -mkdir -p ${TASKCLUSTER_ARTIFACTS} || true +mkdir -p ${CI_ARTIFACTS_DIR} || true -cp ${DS_DSDIR}/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ +cp ${DS_DSDIR}/tensorflow/bazel*.log ${CI_ARTIFACTS_DIR}/ package_native_client "native_client.tar.xz" package_libstt_as_zip "libstt.zip" if [ -d ${DS_DSDIR}/wheels ]; then - cp ${DS_DSDIR}/wheels/* ${TASKCLUSTER_ARTIFACTS}/ - cp ${DS_DSDIR}/native_client/javascript/stt-*.tgz ${TASKCLUSTER_ARTIFACTS}/ + cp ${DS_DSDIR}/wheels/* ${CI_ARTIFACTS_DIR}/ + cp ${DS_DSDIR}/native_client/javascript/stt-*.tgz ${CI_ARTIFACTS_DIR}/ fi; if [ -f ${DS_DSDIR}/native_client/javascript/wrapper.tar.gz ]; then - cp ${DS_DSDIR}/native_client/javascript/wrapper.tar.gz ${TASKCLUSTER_ARTIFACTS}/ + cp ${DS_DSDIR}/native_client/javascript/wrapper.tar.gz ${CI_ARTIFACTS_DIR}/ fi; diff --git a/ci_scripts/tf-package.sh b/ci_scripts/tf-package.sh index c4498303..3a17672f 100755 --- a/ci_scripts/tf-package.sh +++ b/ci_scripts/tf-package.sh @@ -4,9 +4,9 @@ set -xe source $(dirname $0)/tf-vars.sh -mkdir -p ${TASKCLUSTER_ARTIFACTS} || true +mkdir -p ${CI_ARTIFACTS_DIR} || true -cp ${DS_ROOT_TASK}/tensorflow/bazel_*.log ${TASKCLUSTER_ARTIFACTS} || true +cp ${DS_ROOT_TASK}/tensorflow/bazel_*.log ${CI_ARTIFACTS_DIR} || true OUTPUT_ROOT="${DS_ROOT_TASK}/tensorflow/bazel-bin" @@ -19,12 +19,12 @@ for output_bin in \ tensorflow/lite/toco/toco; do if [ -f "${OUTPUT_ROOT}/${output_bin}" ]; then - cp ${OUTPUT_ROOT}/${output_bin} ${TASKCLUSTER_ARTIFACTS}/ + cp ${OUTPUT_ROOT}/${output_bin} ${CI_ARTIFACTS_DIR}/ fi; done; if [ -f "${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model" ]; then - cp ${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model ${TASKCLUSTER_ARTIFACTS}/lite_benchmark_model + cp ${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model ${CI_ARTIFACTS_DIR}/lite_benchmark_model fi # It seems that bsdtar and gnutar are behaving a bit differently on the way @@ -42,21 +42,21 @@ fi; # - /Users/build-user/TaskCluster/HeavyTasks/X/ (OSX) # - C:\builds\tc-workdir\ (windows) -if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then +if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then export PATH=$PATH:'/c/Program Files/7-Zip/' pushd ${DS_ROOT_TASK} - 7z a '-xr!.\dls\' '-xr!.\tmp\' '-xr!.\msys64\' -snl -snh -so home.tar . | 7z a -si ${TASKCLUSTER_ARTIFACTS}/home.tar.xz + 7z a '-xr!.\dls\' '-xr!.\tmp\' '-xr!.\msys64\' -snl -snh -so home.tar . | 7z a -si ${CI_ARTIFACTS_DIR}/home.tar.xz popd else - ${TAR} -C ${DS_ROOT_TASK} ${TAR_EXCLUDE} -cf - . | ${XZ} > ${TASKCLUSTER_ARTIFACTS}/home.tar.xz + ${TAR} -C ${DS_ROOT_TASK} ${TAR_EXCLUDE} -cf - . | ${XZ} > ${CI_ARTIFACTS_DIR}/home.tar.xz fi if [ "${OS}" = "Linux" ]; then SHA_SUM_GEN="sha256sum" -elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then +elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then SHA_SUM_GEN="sha256sum" elif [ "${OS}" = "Darwin" ]; then SHA_SUM_GEN="shasum -a 256" fi; -${SHA_SUM_GEN} ${TASKCLUSTER_ARTIFACTS}/* > ${TASKCLUSTER_ARTIFACTS}/checksums.txt +${SHA_SUM_GEN} ${CI_ARTIFACTS_DIR}/* > ${CI_ARTIFACTS_DIR}/checksums.txt diff --git a/ci_scripts/tf-setup.sh b/ci_scripts/tf-setup.sh index 1cf7c841..9c41952a 100755 --- a/ci_scripts/tf-setup.sh +++ b/ci_scripts/tf-setup.sh @@ -51,7 +51,7 @@ elif [ "${OS}" = "Darwin" ]; then fi; mkdir -p ${DS_ROOT_TASK}/bin || true pushd ${DS_ROOT_TASK}/bin - if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel.exe else /bin/bash ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${BAZEL_INSTALL_FLAGS} @@ -104,7 +104,7 @@ if [ ! -z "${install_android}" ]; then popd fi -mkdir -p ${TASKCLUSTER_ARTIFACTS} || true +mkdir -p ${CI_ARTIFACTS_DIR} || true # Taken from https://www.tensorflow.org/install/source diff --git a/ci_scripts/tf-vars.sh b/ci_scripts/tf-vars.sh index 0e6014c9..a34a61eb 100755 --- a/ci_scripts/tf-vars.sh +++ b/ci_scripts/tf-vars.sh @@ -26,25 +26,28 @@ if [ "${OS}" = "Linux" ]; then WGET=/usr/bin/wget TAR=tar XZ="pixz -9" -elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then - if [ -z "${TASKCLUSTER_TASK_DIR}" -o -z "${TASKCLUSTER_ARTIFACTS}" ]; then +elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then + if [ -z "${CI_TASK_DIR}" -o -z "${CI_ARTIFACTS_DIR}" ]; then echo "Inconsistent Windows setup: missing some vars." - echo "TASKCLUSTER_TASK_DIR=${TASKCLUSTER_TASK_DIR}" - echo "TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS}" + echo "CI_TASK_DIR=${CI_TASK_DIR}" + echo "CI_ARTIFACTS_DIR=${CI_ARTIFACTS_DIR}" exit 1 fi; # Re-export with cygpath to make sure it is sane, otherwise it might trigger # unobvious failures with cp etc. - export TASKCLUSTER_TASK_DIR="$(cygpath ${TASKCLUSTER_TASK_DIR})" - export TASKCLUSTER_ARTIFACTS="$(cygpath ${TASKCLUSTER_ARTIFACTS})" + export CI_TASK_DIR="$(cygpath ${CI_TASK_DIR})" + export CI_ARTIFACTS_DIR="$(cygpath ${CI_ARTIFACTS_DIR})" - export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} + export DS_ROOT_TASK=${CI_TASK_DIR} + export BAZEL_VC='C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC' + export BAZEL_SH='C:\builds\tc-workdir\msys64\usr\bin\bash' + export TC_WIN_BUILD_PATH='C:\builds\tc-workdir\msys64\usr\bin;C:\Python36' export MSYS2_ARG_CONV_EXCL='//' - mkdir -p ${TASKCLUSTER_TASK_DIR}/tmp/ - export TEMP=${TASKCLUSTER_TASK_DIR}/tmp/ - export TMP=${TASKCLUSTER_TASK_DIR}/tmp/ + mkdir -p ${CI_TASK_DIR}/tmp/ + export TEMP=${CI_TASK_DIR}/tmp/ + export TMP=${CI_TASK_DIR}/tmp/ BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-windows-x86_64.exe BAZEL_SHA256=776db1f4986dacc3eda143932f00f7529f9ee65c7c1c004414c44aaa6419d0e9 @@ -56,14 +59,14 @@ elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then TAR=/usr/bin/tar.exe XZ="xz -9 -T0" elif [ "${OS}" = "Darwin" ]; then - if [ -z "${TASKCLUSTER_TASK_DIR}" -o -z "${TASKCLUSTER_ARTIFACTS}" ]; then + if [ -z "${CI_TASK_DIR}" -o -z "${CI_ARTIFACTS_DIR}" ]; then echo "Inconsistent OSX setup: missing some vars." - echo "TASKCLUSTER_TASK_DIR=${TASKCLUSTER_TASK_DIR}" - echo "TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS}" + echo "CI_TASK_DIR=${CI_TASK_DIR}" + echo "CI_ARTIFACTS_DIR=${CI_ARTIFACTS_DIR}" exit 1 fi; - export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} + export DS_ROOT_TASK=${CI_TASK_DIR} BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-darwin-x86_64.sh BAZEL_SHA256=5cfa97031b43432b3c742c80e2e01c41c0acdca7ba1052fc8cf1e291271bc9cd @@ -76,7 +79,7 @@ fi; # /tmp/artifacts for docker-worker on linux, # and task subdir for generic-worker on osx -export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts} +export CI_ARTIFACTS_DIR=${CI_ARTIFACTS_DIR:-/tmp/artifacts} ### Define variables that needs to be exported to other processes @@ -95,7 +98,7 @@ fi; export TF_ENABLE_XLA=0 if [ "${OS}" = "Linux" ]; then TF_NEED_JEMALLOC=1 -elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then +elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then TF_NEED_JEMALLOC=0 elif [ "${OS}" = "Darwin" ]; then TF_NEED_JEMALLOC=0 @@ -120,8 +123,6 @@ if [ "${OS}" = "Linux" ]; then source /etc/os-release if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then export PYTHON_BIN_PATH=/usr/bin/python3 - else - export PYTHON_BIN_PATH=/usr/bin/python2.7 fi elif [ "${OS}" != "${TC_MSYS_VERSION}" ]; then export PYTHON_BIN_PATH=/usr/bin/python2.7 @@ -138,7 +139,7 @@ fi # Build for generic amd64 platforms, no device-specific optimization # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for targetting specific CPUs -if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then +if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then OPT_FLAGS="/arch:AVX" else OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx" @@ -161,7 +162,7 @@ export BAZEL_OUTPUT_USER_ROOT NVCC_COMPUTE="3.5" ### Define build parameters/env variables that we will re-ues in sourcing scripts. -if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then +if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_PATHS=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\"" else TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${DS_ROOT_TASK}/STT/CUDA\" TF_CUDA_PATHS=\"${DS_ROOT_TASK}/STT/CUDA\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\"" @@ -179,7 +180,7 @@ fi BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV" -if [ "${OS}" != "${TC_MSYS_VERSION}" ]; then +if [ "${OS}" != "${CI_MSYS_VERSION}" ]; then BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl --copt=-fvisibility=hidden" fi