experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

CI rebrand pass

Browse Source
This commit is contained in:
Reuben Morais 2021-04-10 16:29:52 +02:00
parent 7855f0a563
commit 8fe4eb8357
28 changed files with 193 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/actions/python-build/action.yml vendored
View File

@ -33,9 +33,9 @@ runs:
mkdir -p wheels
shell: bash
- run: |
PROJECT_NAME="deepspeech"
PROJECT_NAME="stt"
if [ "${{ inputs.build_flavor }}" = "tflite" ]; then
PROJECT_NAME="deepspeech-tflite"
PROJECT_NAME="stt-tflite"
fi
NUMPY_BUILD_VERSION="${{ inputs.numpy_build }}" \

57
.github/workflows/macOS-amd64.yml vendored
View File

@ -135,8 +135,13 @@ jobs:
./bin/run-ci-ldc93s1_new.sh 249 ${bits}
./bin/run-ci-ldc93s1_tflite.sh ${bits}
- run: |
curl -vsSL https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/macOS.amd64.convert_graphdef_memmapped_format.xz | xz -d > /tmp/convert_graphdef_memmapped_format
- name: Download convert_graphdef_memmapped_format tool
run: |
set -ex
wget -O temp.zip https://github.com/coqui-ai/STT/releases/download/v0.9.3/convert_graphdef_memmapped_format.macOS.amd64.zip
unzip temp.zip
rm temp.zip
mv convert_graphdef_memmapped_format /tmp
chmod +x /tmp/convert_graphdef_memmapped_format
/tmp/convert_graphdef_memmapped_format --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm
- run: |
@ -208,7 +213,7 @@ jobs:
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
build-lib_macOS:
name: "Build libdeepspeech.so and deepspeech C++ binary"
name: "Build libstt+client"
runs-on: macos-10.15
needs: [ build-tensorflow-macOS, tensorflow_opt-macOS ]
strategy:
@ -247,8 +252,8 @@ jobs:
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libdeepspeech.${{ matrix.build-flavor }}.zip"
path: ${{ github.workspace }}/artifacts/libdeepspeech.zip
name: "libstt.${{ matrix.build-flavor }}.zip"
path: ${{ github.workspace }}/artifacts/libstt.zip
build-python-macOS:
name: "Build python bindings for macOS"
runs-on: macos-10.15
@ -296,7 +301,7 @@ jobs:
numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}"
- uses: actions/upload-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
path: ${{ github.workspace }}/wheels/*.whl
build-nodejs-macOS:
name: "Build NodeJS and ElectronJS for macOS"
@ -339,8 +344,8 @@ jobs:
path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz
- uses: actions/upload-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz
name: "stt-${{ matrix.build-flavor }}.tgz"
path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz
test-cpp-macOS:
name: "Test C++ binary on macOS"
runs-on: macos-10.15
@ -353,9 +358,9 @@ jobs:
bitrate: ["8k", "16k"]
env:
TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
@ -395,9 +400,9 @@ jobs:
bitrate: ["8k", "16k"]
env:
TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
@ -408,7 +413,7 @@ jobs:
python-version: ${{ matrix.python-version }}
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
path: ${{ env.TASKCLUSTER_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
@ -420,7 +425,7 @@ jobs:
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
pip3 install --only-binary :all: --upgrade ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.whl
pip3 install --only-binary :all: --upgrade ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.whl
- uses: ./.github/actions/run-tests
with:
runtime: "python"
@ -441,9 +446,9 @@ jobs:
bitrate: ["16k"]
env:
TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
@ -454,7 +459,7 @@ jobs:
node-version: ${{ matrix.nodejs-version }}
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
name: "stt-${{ matrix.build-flavor }}.tgz"
path: ${{ env.TASKCLUSTER_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
@ -466,7 +471,7 @@ jobs:
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz
npm install ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.tgz
- uses: ./.github/actions/run-tests
with:
runtime: "node"
@ -486,9 +491,9 @@ jobs:
bitrate: ["16k"]
env:
TASKCLUSTER_TMP_DIR: ${{ github.workspace }}/tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
@ -499,7 +504,7 @@ jobs:
node-version: 12
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
name: "stt-${{ matrix.build-flavor }}.tgz"
path: ${{ env.TASKCLUSTER_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
@ -511,7 +516,7 @@ jobs:
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz
npm install ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.tgz
- run: |
npm install electron@${{ matrix.electronjs-version }}
- uses: ./.github/actions/run-tests

65
.github/workflows/windows-amd64.yml vendored
View File

@ -3,7 +3,7 @@ on:
pull_request:
push:
branches:
- master
- main
env:
TASKCLUSTER_TASK_DIR: ${{ github.workspace }}
TASKCLUSTER_ARTIFACTS: ${{ github.workspace }}/artifacts
@ -180,7 +180,9 @@ jobs:
- run: |
pip install ds_ctcdecoder-*-cp36-cp36m-manylinux1_x86_64.whl
DS_NODECODER=y pip install --upgrade .
- run: |
- name: Run training script
run: |
set -ex
bits=""
if [ "${{ matrix.bitrate }}" = "8k" ]; then
bits=8000
@ -194,12 +196,17 @@ jobs:
echo "Moving ${bits} to LDC93S1.wav"
mv data/smoke_test/LDC93S1_pcms16le_1_${bits}.wav data/smoke_test/LDC93S1.wav
./bin/run-tc-ldc93s1_new.sh 249 ${bits}
./bin/run-ci-ldc93s1_new.sh 249 ${bits}
if [ "${{ matrix.build-flavor }}" = "tflite" ]; then
./bin/run-tc-ldc93s1_tflite.sh ${bits}
./bin/run-ci-ldc93s1_tflite.sh ${bits}
fi
- run: |
curl -vsSL https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/linux.amd64.convert_graphdef_memmapped_format.xz | xz -d > /tmp/convert_graphdef_memmapped_format
- name: Download convert_graphdef_memmapped_format tool
run: |
set -ex
wget -O temp.zip https://github.com/coqui-ai/STT/releases/download/v0.9.3/convert_graphdef_memmapped_format.linux.amd64.zip
unzip temp.zip
rm temp.zip
mv convert_graphdef_memmapped_format /tmp
chmod +x /tmp/convert_graphdef_memmapped_format
/tmp/convert_graphdef_memmapped_format --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm
if: matrix.build-flavor == 'tf'
@ -281,7 +288,7 @@ jobs:
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
build-lib_Windows:
name: "Build libdeepspeech.so and deepspeech C++ binary"
name: "Build libstt+client"
runs-on: windows-2019
needs: [build-tensorflow-Windows, tensorflow_opt-Windows]
strategy:
@ -330,8 +337,8 @@ jobs:
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libdeepspeech.${{ matrix.build-flavor }}.zip"
path: ${{ github.workspace }}/artifacts/libdeepspeech.zip
name: "libstt.${{ matrix.build-flavor }}.zip"
path: ${{ github.workspace }}/artifacts/libstt.zip
build-python-Windows:
name: "Build Python bindings for Windows"
runs-on: windows-2019
@ -390,7 +397,7 @@ jobs:
numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}"
- uses: actions/upload-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
path: ${{ github.workspace }}/wheels/*.whl
build-nodejs-Windows:
name: "Build NodeJS/ElectronJS for Windows"
@ -445,8 +452,8 @@ jobs:
path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz
- uses: actions/upload-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz
name: "stt-${{ matrix.build-flavor }}.tgz"
path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz
test-cpp-Windows:
name: "Test C++ binary on Windows"
runs-on: windows-2019
@ -456,7 +463,7 @@ jobs:
build-flavor: ["tf", "tflite"]
env:
TASKCLUSTER_TMP_DIR: tmp/
DEEPSPEECH_TEST_MODEL: tmp/output_graph.pb
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
@ -504,9 +511,9 @@ jobs:
bitrate: ["8k", "16k"]
env:
TASKCLUSTER_TMP_DIR: tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: tmp/output_graph.pb
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
@ -524,7 +531,7 @@ jobs:
- uses: ./.github/actions/win-install-sox
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl"
path: ${{ env.TASKCLUSTER_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
@ -536,7 +543,7 @@ jobs:
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
python -m pip install --only-binary :all: --upgrade ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.whl
python -m pip install --only-binary :all: --upgrade ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.whl
- uses: ./.github/actions/run-tests
with:
runtime: "python"
@ -556,9 +563,9 @@ jobs:
bitrate: ["16k"]
env:
TASKCLUSTER_TMP_DIR: tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: tmp/output_graph.pb
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
@ -576,7 +583,7 @@ jobs:
- uses: ./.github/actions/win-install-sox
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
name: "stt-${{ matrix.build-flavor }}.tgz"
path: ${{ env.TASKCLUSTER_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
@ -588,7 +595,7 @@ jobs:
if: matrix.models == 'test'
- run: |
ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz
npm install ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.tgz
- uses: ./.github/actions/run-tests
with:
runtime: "node"
@ -608,9 +615,9 @@ jobs:
bitrate: ["16k"]
env:
TASKCLUSTER_TMP_DIR: tmp/
DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb
DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm
DEEPSPEECH_TEST_MODEL: tmp/output_graph.pb
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
@ -628,7 +635,7 @@ jobs:
- uses: ./.github/actions/win-install-sox
- uses: actions/download-artifact@v2
with:
name: "deepspeech-${{ matrix.build-flavor }}.tgz"
name: "stt-${{ matrix.build-flavor }}.tgz"
path: ${{ env.TASKCLUSTER_TMP_DIR }}
- uses: actions/download-artifact@v2
with:
@ -638,10 +645,10 @@ jobs:
- run: |
ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
if: matrix.models == 'test'
- name: Install deepspeech NodeJS package
- name: Install STT NodeJS package
run: |
ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/
npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz
npm install ${{ env.TASKCLUSTER_TMP_DIR }}/stt*.tgz
- run: |
npm install electron@${{ matrix.electronjs-version }}
- uses: ./.github/actions/run-tests

2
ci_scripts/all-utils.sh
View File

@ -85,7 +85,7 @@ verify_bazel_rebuild()
cp ${DS_DSDIR}/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|[for host]|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libdeepspeech.so' | wc -l)
spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|[for host]|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libstt.so' | wc -l)
if [ "${spurious_rebuilds}" -ne 0 ]; then
echo "Bazel rebuilds some file it should not, please check."

8
ci_scripts/all-vars.sh
View File

@ -54,11 +54,11 @@ export DS_TFDIR=${DS_ROOT_TASK}/tensorflow
export DS_DSDIR=${DS_ROOT_TASK}/
export DS_EXAMPLEDIR=${DS_ROOT_TASK}/examples
export DS_VERSION="$(cat ${DS_DSDIR}/training/deepspeech_training/VERSION)"
export DS_VERSION="$(cat ${DS_DSDIR}/training/coqui_stt_training/VERSION)"
export GRADLE_USER_HOME=${DS_ROOT_TASK}/gradle-cache
export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/
export ANDROID_NDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/android-ndk-r18b/
export ANDROID_SDK_HOME=${DS_ROOT_TASK}/STT/Android/SDK/
export ANDROID_NDK_HOME=${DS_ROOT_TASK}/STT/Android/android-ndk-r18b/
WGET=${WGET:-"wget"}
TAR=${TAR:-"tar"}
@ -78,7 +78,7 @@ if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
UNXZ="xz -9 -T0 -d"
fi
model_source="${DEEPSPEECH_TEST_MODEL}"
model_source="${STT_TEST_MODEL}"
model_name="$(basename "${model_source}")"
model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm"
model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}"

94
ci_scripts/asserts.sh
View File

@ -251,9 +251,9 @@ assert_tensorflow_version()
assert_shows_something "$1" "${EXPECTED_TENSORFLOW_VERSION}"
}
assert_deepspeech_version()
assert_stt_version()
{
assert_not_present "$1" "DeepSpeech: unknown"
assert_not_present "$1" "Coqui STT: unknown"
}
# We need to ensure that running on inference really leverages GPU because
@ -261,7 +261,7 @@ assert_deepspeech_version()
ensure_cuda_usage()
{
local _maybe_cuda=$1
DS_BINARY_FILE=${DS_BINARY_FILE:-"deepspeech"}
DS_BINARY_FILE=${DS_BINARY_FILE:-"stt"}
if [ "${_maybe_cuda}" = "cuda" ]; then
set +e
@ -278,19 +278,19 @@ ensure_cuda_usage()
check_versions()
{
set +e
ds_help=$(${DS_BINARY_PREFIX}deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>&1 1>/dev/null)
ds_help=$(${DS_BINARY_PREFIX}stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>&1 1>/dev/null)
set -e
assert_tensorflow_version "${ds_help}"
assert_deepspeech_version "${ds_help}"
assert_stt_version "${ds_help}"
}
assert_deepspeech_runtime()
assert_stt_runtime()
{
local expected_runtime=$1
set +e
local ds_version=$(${DS_BINARY_PREFIX}deepspeech --version 2>&1)
local ds_version=$(${DS_BINARY_PREFIX}stt --version 2>&1)
set -e
assert_shows_something "${ds_version}" "${expected_runtime}"
@ -298,23 +298,23 @@ assert_deepspeech_runtime()
check_runtime_nodejs()
{
assert_deepspeech_runtime "Runtime: Node"
assert_stt_runtime "Runtime: Node"
}
check_runtime_electronjs()
{
assert_deepspeech_runtime "Runtime: Electron"
assert_stt_runtime "Runtime: Electron"
}
run_tflite_basic_inference_tests()
{
set +e
phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}deepspeech --model ${DATA_TMP_DIR}/${model_name} --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$?"
set +e
phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}deepspeech --model ${DATA_TMP_DIR}/${model_name} --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$?"
}
@ -322,22 +322,22 @@ run_tflite_basic_inference_tests()
run_netframework_inference_tests()
{
set +e
phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(STTConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?"
set +e
phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended yes 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(STTConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended yes 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?"
set +e
phrase_pbmodel_nolm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(STTConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?"
set +e
phrase_pbmodel_withlm=$(DeepSpeechConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm=$(STTConsole.exe --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_working_ldc93s1_lm "${phrase_pbmodel_withlm}" "$?"
}
@ -345,22 +345,22 @@ run_netframework_inference_tests()
run_electronjs_inference_tests()
{
set +e
phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?"
set +e
phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?"
set +e
phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_working_ldc93s1 "${phrase_pbmodel_nolm}" "$?"
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
set -e
assert_working_ldc93s1_lm "${phrase_pbmodel_withlm}" "$?"
}
@ -368,30 +368,30 @@ run_electronjs_inference_tests()
run_basic_inference_tests()
{
set +e
deepspeech --model "" --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr
stt --model "" --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr
set -e
grep "Missing model information" ${TASKCLUSTER_TMP_DIR}/stderr
set +e
phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$status"
set +e
phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --extended 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$status"
set +e
phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1 "${phrase_pbmodel_nolm}" "$status"
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" "$status"
@ -402,13 +402,13 @@ run_all_inference_tests()
run_basic_inference_tests
set +e
phrase_pbmodel_nolm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_nolm_stereo_44k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1 "${phrase_pbmodel_nolm_stereo_44k}" "$status"
set +e
phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm_stereo_44k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm_stereo_44k}" "$status"
@ -416,12 +416,12 @@ run_all_inference_tests()
# Run down-sampling warning test only when we actually perform downsampling
if [ "${ldc93s1_sample_filename}" != "LDC93S1_pcms16le_1_8000.wav" ]; then
set +e
phrase_pbmodel_nolm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
phrase_pbmodel_nolm_mono_8k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
set -e
assert_correct_warning_upsampling "${phrase_pbmodel_nolm_mono_8k}"
set +e
phrase_pbmodel_withlm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
phrase_pbmodel_withlm_mono_8k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
set -e
assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}"
fi;
@ -452,19 +452,19 @@ run_prod_inference_tests()
local _bitrate=$1
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
set +e
phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm_stereo_44k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodmodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status"
@ -472,7 +472,7 @@ run_prod_inference_tests()
# Run down-sampling warning test only when we actually perform downsampling
if [ "${ldc93s1_sample_filename}" != "LDC93S1_pcms16le_1_8000.wav" ]; then
set +e
phrase_pbmodel_withlm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
phrase_pbmodel_withlm_mono_8k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
set -e
assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}"
fi;
@ -483,19 +483,19 @@ run_prodtflite_inference_tests()
local _bitrate=$1
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
set +e
phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
phrase_pbmodel_withlm_stereo_44k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodtflitemodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status"
@ -503,7 +503,7 @@ run_prodtflite_inference_tests()
# Run down-sampling warning test only when we actually perform downsampling
if [ "${ldc93s1_sample_filename}" != "LDC93S1_pcms16le_1_8000.wav" ]; then
set +e
phrase_pbmodel_withlm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
phrase_pbmodel_withlm_mono_8k=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
set -e
assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}"
fi;
@ -512,13 +512,13 @@ run_prodtflite_inference_tests()
run_multi_inference_tests()
{
set +e -o pipefail
multi_phrase_pbmodel_nolm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/ 2>${TASKCLUSTER_TMP_DIR}/stderr | tr '\n' '%')
multi_phrase_pbmodel_nolm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --audio ${TASKCLUSTER_TMP_DIR}/ 2>${TASKCLUSTER_TMP_DIR}/stderr | tr '\n' '%')
status=$?
set -e +o pipefail
assert_correct_multi_ldc93s1 "${multi_phrase_pbmodel_nolm}" "$status"
set +e -o pipefail
multi_phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/ 2>${TASKCLUSTER_TMP_DIR}/stderr | tr '\n' '%')
multi_phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/ 2>${TASKCLUSTER_TMP_DIR}/stderr | tr '\n' '%')
status=$?
set -e +o pipefail
assert_correct_multi_ldc93s1 "${multi_phrase_pbmodel_withlm}" "$status"
@ -526,7 +526,7 @@ run_multi_inference_tests()
run_hotword_tests()
{
DS_BINARY_FILE=${DS_BINARY_FILE:-"deepspeech"}
DS_BINARY_FILE=${DS_BINARY_FILE:-"stt"}
set +e
hotwords_decode=$(${DS_BINARY_PREFIX}${DS_BINARY_FILE} --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --hot_words "foo:0.0,bar:-0.1" 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
@ -537,7 +537,7 @@ run_hotword_tests()
run_android_hotword_tests()
{
set +e
hotwords_decode=$(${DS_BINARY_PREFIX}deepspeech --model ${DATA_TMP_DIR}/${model_name} --scorer ${DATA_TMP_DIR}/kenlm.scorer --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} --hot_words "foo:0.0,bar:-0.1" 2>${TASKCLUSTER_TMP_DIR}/stderr)
hotwords_decode=$(${DS_BINARY_PREFIX}stt --model ${DATA_TMP_DIR}/${model_name} --scorer ${DATA_TMP_DIR}/kenlm.scorer --audio ${DATA_TMP_DIR}/${ldc93s1_sample_filename} --hot_words "foo:0.0,bar:-0.1" 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_lm "${hotwords_decode}" "$status"
@ -546,7 +546,7 @@ run_android_hotword_tests()
run_cpp_only_inference_tests()
{
set +e
phrase_pbmodel_withlm_intermediate_decode=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 1280 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
phrase_pbmodel_withlm_intermediate_decode=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 1280 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
status=$?
set -e
assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm_intermediate_decode}" "$status"
@ -555,13 +555,13 @@ run_cpp_only_inference_tests()
run_js_streaming_inference_tests()
{
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
status=$?
set -e
assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" "$status"
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
status=$?
set -e
assert_correct_ldc93s1_lm "${phrase_pbmodel_withlm}" "$status"
@ -571,14 +571,14 @@ run_js_streaming_prod_inference_tests()
{
local _bitrate=$1
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
status=$?
set -e
assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
local _bitrate=$1
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
status=$?
set -e
assert_correct_ldc93s1_prodmodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
@ -588,14 +588,14 @@ run_js_streaming_prodtflite_inference_tests()
{
local _bitrate=$1
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
status=$?
set -e
assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"
local _bitrate=$1
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
phrase_pbmodel_withlm=$(stt --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --scorer ${TASKCLUSTER_TMP_DIR}/kenlm.scorer --audio ${TASKCLUSTER_TMP_DIR}/${ldc93s1_sample_filename} --stream --extended 2>${TASKCLUSTER_TMP_DIR}/stderr | tail -n 1)
status=$?
set -e
assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status" "${_bitrate}"

4
ci_scripts/build-utils.sh
View File

@ -32,7 +32,7 @@ shutdown_bazel()
bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown
}
do_deepspeech_binary_build()
do_stt_binary_build()
{
cd ${DS_DSDIR}
make -C native_client/ \
@ -42,5 +42,5 @@ do_deepspeech_binary_build()
EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \
EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \
EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \
deepspeech${PLATFORM_EXE_SUFFIX}
stt${PLATFORM_EXE_SUFFIX}
}

4
ci_scripts/cpp-tests-prod.sh
View File

@ -9,10 +9,10 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_PROD_MODEL}
model_source=${STT_PROD_MODEL}
model_name=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP}
model_source_mmap=${STT_PROD_MODEL_MMAP}
model_name_mmap=$(basename "${model_source_mmap}")
download_model_prod

4
ci_scripts/cpp_tflite-tests-prod.sh
View File

@ -9,10 +9,10 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
model_source=${STT_PROD_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite}
model_source_mmap=${STT_PROD_MODEL_MMAP//.pbmm/.tflite}
export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR}
download_model_prod

2
ci_scripts/cpp_tflite-tests.sh
View File

@ -9,7 +9,7 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_source=${STT_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR}

2
ci_scripts/cpp_tflite_basic-tests.sh
View File

@ -9,7 +9,7 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_source=${STT_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR}

2
ci_scripts/cppwin_tflite-tests.sh
View File

@ -9,7 +9,7 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_source=${STT_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR}

2
ci_scripts/electronjs-tests.sh
View File

@ -23,7 +23,7 @@ which node
node --version
deepspeech --version
stt --version
check_runtime_electronjs

4
ci_scripts/electronjs_tflite-tests.sh
View File

@ -9,7 +9,7 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_source=${STT_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
@ -27,7 +27,7 @@ which node
node --version
deepspeech --version
stt --version
check_runtime_electronjs

4
ci_scripts/host-build.sh
View File

@ -11,7 +11,7 @@ source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libdeepspeech.so
//native_client:libstt.so
//native_client:generate_scorer_package
"
@ -25,4 +25,4 @@ SYSTEM_TARGET=host
do_bazel_build
do_deepspeech_binary_build
do_stt_binary_build

4
ci_scripts/node-tests-prod.sh
View File

@ -9,10 +9,10 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_PROD_MODEL}
model_source=${STT_PROD_MODEL}
model_name=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP}
model_source_mmap=${STT_PROD_MODEL_MMAP}
model_name_mmap=$(basename "${model_source_mmap}")
download_model_prod

4
ci_scripts/node_tflite-tests-prod.sh
View File

@ -9,10 +9,10 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
model_source=${STT_PROD_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite}
model_source_mmap=${STT_PROD_MODEL_MMAP//.pbmm/.tflite}
download_model_prod

2
ci_scripts/node_tflite-tests.sh
View File

@ -9,7 +9,7 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_source=${STT_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")

42
ci_scripts/package-utils.sh
View File

@ -5,14 +5,14 @@ set -xe
package_native_client()
{
tensorflow_dir=${DS_TFDIR}
deepspeech_dir=${DS_DSDIR}
stt_dir=${DS_DSDIR}
artifacts_dir=${TASKCLUSTER_ARTIFACTS}
artifact_name=$1
if [ ! -d ${tensorflow_dir} -o ! -d ${deepspeech_dir} -o ! -d ${artifacts_dir} ]; then
if [ ! -d ${tensorflow_dir} -o ! -d ${stt_dir} -o ! -d ${artifacts_dir} ]; then
echo "Missing directory. Please check:"
echo "tensorflow_dir=${tensorflow_dir}"
echo "deepspeech_dir=${deepspeech_dir}"
echo "stt_dir=${stt_dir}"
echo "artifacts_dir=${artifacts_dir}"
exit 1
fi;
@ -22,32 +22,32 @@ package_native_client()
fi;
win_lib=""
if [ -f "${tensorflow_dir}/bazel-bin/native_client/libdeepspeech.so.if.lib" ]; then
win_lib="-C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so.if.lib"
if [ -f "${tensorflow_dir}/bazel-bin/native_client/libstt.so.if.lib" ]; then
win_lib="-C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so.if.lib"
fi;
${TAR} -cf - \
-C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so \
${win_lib} \
-C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \
-C ${deepspeech_dir}/ LICENSE \
-C ${deepspeech_dir}/native_client/ deepspeech${PLATFORM_EXE_SUFFIX} \
-C ${deepspeech_dir}/native_client/ deepspeech.h \
-C ${deepspeech_dir}/native_client/kenlm/ README.mozilla \
-C ${stt_dir}/ LICENSE \
-C ${stt_dir}/native_client/ stt${PLATFORM_EXE_SUFFIX} \
-C ${stt_dir}/native_client/ coqui-stt.h \
-C ${stt_dir}/native_client/kenlm/ README.coqui \
| ${XZ} > "${artifacts_dir}/${artifact_name}"
}
package_native_client_ndk()
{
deepspeech_dir=${DS_DSDIR}
stt_dir=${DS_DSDIR}
tensorflow_dir=${DS_TFDIR}
artifacts_dir=${TASKCLUSTER_ARTIFACTS}
artifact_name=$1
arch_abi=$2
if [ ! -d ${deepspeech_dir} -o ! -d ${artifacts_dir} ]; then
if [ ! -d ${stt_dir} -o ! -d ${artifacts_dir} ]; then
echo "Missing directory. Please check:"
echo "deepspeech_dir=${deepspeech_dir}"
echo "stt_dir=${stt_dir}"
echo "artifacts_dir=${artifacts_dir}"
exit 1
fi;
@ -61,17 +61,17 @@ package_native_client_ndk()
fi;
${TAR} -cf - \
-C ${deepspeech_dir}/native_client/libs/${arch_abi}/ deepspeech \
-C ${deepspeech_dir}/native_client/libs/${arch_abi}/ libdeepspeech.so \
-C ${stt_dir}/native_client/libs/${arch_abi}/ stt \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libstt.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \
-C ${deepspeech_dir}/native_client/libs/${arch_abi}/ libc++_shared.so \
-C ${deepspeech_dir}/native_client/ deepspeech.h \
-C ${deepspeech_dir}/ LICENSE \
-C ${deepspeech_dir}/native_client/kenlm/ README.mozilla \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libc++_shared.so \
-C ${stt_dir}/native_client/ coqui-stt.h \
-C ${stt_dir}/ LICENSE \
-C ${stt_dir}/native_client/kenlm/ README.coqui \
| ${XZ} > "${artifacts_dir}/${artifact_name}"
}
package_libdeepspeech_as_zip()
package_libstt_as_zip()
{
tensorflow_dir=${DS_TFDIR}
artifacts_dir=${TASKCLUSTER_ARTIFACTS}
@ -88,5 +88,5 @@ package_libdeepspeech_as_zip()
echo "Please specify artifact name."
fi;
${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" ${tensorflow_dir}/bazel-bin/native_client/libdeepspeech.so
${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" ${tensorflow_dir}/bazel-bin/native_client/libstt.so
}

4
ci_scripts/package.sh
View File

@ -11,11 +11,11 @@ cp ${DS_DSDIR}/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
package_native_client "native_client.tar.xz"
package_libdeepspeech_as_zip "libdeepspeech.zip"
package_libstt_as_zip "libstt.zip"
if [ -d ${DS_DSDIR}/wheels ]; then
cp ${DS_DSDIR}/wheels/* ${TASKCLUSTER_ARTIFACTS}/
cp ${DS_DSDIR}/native_client/javascript/deepspeech-*.tgz ${TASKCLUSTER_ARTIFACTS}/
cp ${DS_DSDIR}/native_client/javascript/stt-*.tgz ${TASKCLUSTER_ARTIFACTS}/
fi;
if [ -f ${DS_DSDIR}/native_client/javascript/wrapper.tar.gz ]; then

8
ci_scripts/python-tests-prod.sh
View File

@ -9,18 +9,18 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_PROD_MODEL}
model_source=${STT_PROD_MODEL}
model_name=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP}
model_source_mmap=${STT_PROD_MODEL_MMAP}
model_name_mmap=$(basename "${model_source_mmap}")
download_model_prod
download_material
which deepspeech
deepspeech --version
which stt
stt --version
run_prod_inference_tests "${bitrate}"

4
ci_scripts/python-tests.sh
View File

@ -11,8 +11,8 @@ set_ldc_sample_filename "${bitrate}"
download_data
which deepspeech
deepspeech --version
which stt
stt --version
run_all_inference_tests

8
ci_scripts/python_tflite-tests-prod.sh
View File

@ -9,16 +9,16 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
model_source=${STT_PROD_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite}
model_source_mmap=${STT_PROD_MODEL_MMAP//.pbmm/.tflite}
download_model_prod
download_material
which deepspeech
deepspeech --version
which stt
stt --version
run_prodtflite_inference_tests "${bitrate}"

6
ci_scripts/python_tflite-tests.sh
View File

@ -9,14 +9,14 @@ source $(dirname "$0")/asserts.sh
bitrate=$1
set_ldc_sample_filename "${bitrate}"
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_source=${STT_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
download_data
which deepspeech
deepspeech --version
which stt
stt --version
run_all_inference_tests

2
ci_scripts/tf-build.sh
View File

@ -23,7 +23,7 @@ pushd ${DS_ROOT_TASK}/tensorflow/
case "$1" in
"--windows-cpu")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIBDEEPSPEECH} ${BUILD_TARGET_LITE_LIB} --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh"
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIBSTT} ${BUILD_TARGET_LITE_LIB} --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh"
;;
"--linux-cpu"|"--darwin-cpu")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}

4
ci_scripts/tf-package.sh
View File

@ -28,9 +28,9 @@ if [ -f "${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model" ]; then
fi
# It seems that bsdtar and gnutar are behaving a bit differently on the way
# they deal with --exclude="./public/*" ; this caused ./DeepSpeech/tensorflow/core/public/
# they deal with --exclude="./public/*" ; this caused ./STT/tensorflow/core/public/
# to be ditched when we just wanted to get rid of ./public/ on OSX.
# Switching to gnutar (already needed for the --transform on DeepSpeech tasks)
# Switching to gnutar (already needed for the --transform on STT tasks)
# does the trick.
TAR_EXCLUDE="--exclude=./dls/*"
if [ "${OS}" = "Darwin" ]; then

18
ci_scripts/tf-setup.sh
View File

@ -65,22 +65,22 @@ bazel shutdown
if [ ! -z "${install_cuda}" ]; then
# Install CUDA and CuDNN
mkdir -p ${DS_ROOT_TASK}/DeepSpeech/CUDA/ || true
mkdir -p ${DS_ROOT_TASK}/STT/CUDA/ || true
pushd ${DS_ROOT_TASK}
CUDA_FILE=`basename ${CUDA_URL}`
PERL5LIB=. sh ${DS_ROOT_TASK}/dls/${CUDA_FILE} --silent --override --toolkit --toolkitpath=${DS_ROOT_TASK}/DeepSpeech/CUDA/ --defaultroot=${DS_ROOT_TASK}/DeepSpeech/CUDA/
PERL5LIB=. sh ${DS_ROOT_TASK}/dls/${CUDA_FILE} --silent --override --toolkit --toolkitpath=${DS_ROOT_TASK}/STT/CUDA/ --defaultroot=${DS_ROOT_TASK}/STT/CUDA/
CUDNN_FILE=`basename ${CUDNN_URL}`
tar xvf ${DS_ROOT_TASK}/dls/${CUDNN_FILE} --strip-components=1 -C ${DS_ROOT_TASK}/DeepSpeech/CUDA/
tar xvf ${DS_ROOT_TASK}/dls/${CUDNN_FILE} --strip-components=1 -C ${DS_ROOT_TASK}/STT/CUDA/
popd
LD_LIBRARY_PATH=${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/:${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/:$LD_LIBRARY_PATH
LD_LIBRARY_PATH=${DS_ROOT_TASK}/STT/CUDA/lib64/:${DS_ROOT_TASK}/STT/CUDA/lib64/stubs/:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH
# We might lack libcuda.so.1 symlink, let's fix as upstream does:
# https://github.com/tensorflow/tensorflow/pull/13811/files?diff=split#diff-2352449eb75e66016e97a591d3f0f43dR96
if [ ! -h "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so.1" ]; then
ln -s "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so" "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so.1"
if [ ! -h "${DS_ROOT_TASK}/STT/CUDA/lib64/stubs/libcuda.so.1" ]; then
ln -s "${DS_ROOT_TASK}/STT/CUDA/lib64/stubs/libcuda.so" "${DS_ROOT_TASK}/STT/CUDA/lib64/stubs/libcuda.so.1"
fi;
else
@ -88,15 +88,15 @@ else
fi
if [ ! -z "${install_android}" ]; then
mkdir -p ${DS_ROOT_TASK}/DeepSpeech/Android/SDK || true
mkdir -p ${DS_ROOT_TASK}/STT/Android/SDK || true
ANDROID_NDK_FILE=`basename ${ANDROID_NDK_URL}`
ANDROID_SDK_FILE=`basename ${ANDROID_SDK_URL}`
pushd ${DS_ROOT_TASK}/DeepSpeech/Android
pushd ${DS_ROOT_TASK}/STT/Android
unzip ${DS_ROOT_TASK}/dls/${ANDROID_NDK_FILE}
popd
pushd ${DS_ROOT_TASK}/DeepSpeech/Android/SDK
pushd ${DS_ROOT_TASK}/STT/Android/SDK
unzip ${DS_ROOT_TASK}/dls/${ANDROID_SDK_FILE}
yes | ./tools/bin/sdkmanager --licenses
./tools/bin/sdkmanager --update

10
ci_scripts/tf-vars.sh
View File

@ -87,9 +87,9 @@ fi;
export PATH
if [ "${OS}" = "Linux" ]; then
export LD_LIBRARY_PATH=${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/:${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/:$LD_LIBRARY_PATH
export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/
export ANDROID_NDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/android-ndk-r18b/
export LD_LIBRARY_PATH=${DS_ROOT_TASK}/STT/CUDA/lib64/:${DS_ROOT_TASK}/STT/CUDA/lib64/stubs/:$LD_LIBRARY_PATH
export ANDROID_SDK_HOME=${DS_ROOT_TASK}/STT/Android/SDK/
export ANDROID_NDK_HOME=${DS_ROOT_TASK}/STT/Android/android-ndk-r18b/
fi;
export TF_ENABLE_XLA=0
@ -164,7 +164,7 @@ NVCC_COMPUTE="3.5"
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_PATHS=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\""
else
TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${DS_ROOT_TASK}/DeepSpeech/CUDA\" TF_CUDA_PATHS=\"${DS_ROOT_TASK}/DeepSpeech/CUDA\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\""
TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${DS_ROOT_TASK}/STT/CUDA\" TF_CUDA_PATHS=\"${DS_ROOT_TASK}/STT/CUDA\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\""
fi
BAZEL_ARM_FLAGS="--config=rpi3 --config=rpi3_opt --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_WITH_RUY_GEMV"
@ -196,4 +196,4 @@ BUILD_TARGET_GRAPH_BENCHMARK="//tensorflow/tools/benchmark:benchmark_model"
BUILD_TARGET_TOCO="//tensorflow/lite/toco:toco"
BUILD_TARGET_LITE_BENCHMARK="//tensorflow/lite/tools/benchmark:benchmark_model"
BUILD_TARGET_LITE_LIB="//tensorflow/lite/c:libtensorflowlite_c.so"
BUILD_TARGET_LIBDEEPSPEECH="//native_client:libdeepspeech.so"
BUILD_TARGET_LIBSTT="//native_client:libstt.so"