experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add TFLite prod tests

Browse Source 
Fixes #2614
This commit is contained in:
Alexandre Lissy 2019-12-20 16:45:04 +01:00
parent 3cd79aecdf
commit 89cd481d52
21 changed files with 347 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
taskcluster/tc-cpp_tflite-tests-prod.sh Normal file
View File

@ -0,0 +1,20 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite}
export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT}
export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR}
download_material "${TASKCLUSTER_TMP_DIR}/ds"
export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
check_tensorflow_version
run_prodtflite_inference_tests

35
taskcluster/tc-node_tflite-tests-prod.sh Normal file
View File

@ -0,0 +1,35 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
nodever=$1
if [ -z "${nodever}" ]; then
echo "No node version given, aborting."
exit 1
fi;
model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite}
export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT}
download_data
node --version
npm --version
NODE_ROOT="${DS_ROOT_TASK}/ds-test/"
NODE_CACHE="${DS_ROOT_TASK}/ds-test.cache/"
export NODE_PATH="${NODE_ROOT}/node_modules/"
export PATH="${NODE_ROOT}:${NODE_PATH}/.bin/:$PATH"
deepspeech_npm_url=$(get_dep_npm_pkg_url)
npm install --prefix ${NODE_ROOT} --cache ${NODE_CACHE} ${deepspeech_npm_url}
check_runtime_nodejs
run_prod_inference_tests

51
taskcluster/tc-python_tflite-tests-prod.sh Normal file
View File

@ -0,0 +1,51 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
unset PYTHON_BIN_PATH
unset PYTHONPATH
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
model_source_mmap=${DEEPSPEECH_PROD_MODEL_MMAP//.pbmm/.tflite}
export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT}
download_data
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
PYENV_NAME=deepspeech-test
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "${PYENV_NAME}"
deepspeech_pkg_url=$(get_python_pkg_url "${pyver_pkg}" "${py_unicode_type}" "deepspeech_tflite")
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
which deepspeech
deepspeech --version
run_prod_inference_tests
virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"

36
taskcluster/tc-tests-utils.sh
View File

@ -256,11 +256,21 @@ assert_correct_ldc93s1_prodmodel()
assert_correct_inference "$1" "she had reduce and greasy wash water all year" "$2"
}
assert_correct_ldc93s1_prodtflitemodel()
{
assert_correct_inference "$1" "she had educate in greasy wash were all year" "$2"
}
assert_correct_ldc93s1_prodmodel_stereo_44k()
{
assert_correct_inference "$1" "she had reduce and greasy wash water all year" "$2"
}
assert_correct_ldc93s1_prodtflitemodel_stereo_44k()
{
assert_correct_inference "$1" "she had educate in greasy wash were all year" "$2"
}
assert_correct_warning_upsampling()
{
assert_shows_something "$1" "erratic speech recognition"
@ -465,6 +475,32 @@ run_prod_inference_tests()
assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}"
}
run_prodtflite_inference_tests()
{
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name} --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status"
set +e
phrase_pbmodel_withlm=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodtflitemodel "${phrase_pbmodel_withlm}" "$status"
set +e
phrase_pbmodel_withlm_stereo_44k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_2_44100.wav 2>${TASKCLUSTER_TMP_DIR}/stderr)
status=$?
set -e
assert_correct_ldc93s1_prodtflitemodel_stereo_44k "${phrase_pbmodel_withlm_stereo_44k}" "$status"
set +e
phrase_pbmodel_withlm_mono_8k=$(deepspeech --model ${TASKCLUSTER_TMP_DIR}/${model_name_mmap} --lm ${TASKCLUSTER_TMP_DIR}/lm.binary --trie ${TASKCLUSTER_TMP_DIR}/trie --audio ${TASKCLUSTER_TMP_DIR}/LDC93S1_pcms16le_1_8000.wav 2>&1 1>/dev/null)
set -e
assert_correct_warning_upsampling "${phrase_pbmodel_withlm_mono_8k}"
}
run_multi_inference_tests()
{
set +e -o pipefail

9
taskcluster/test-cpp_tflite-linux-amd64-prod-opt.yml Executable file
View File

@ -0,0 +1,9 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-cpp_tflite-tests-prod.sh"
metadata:
name: "DeepSpeech Linux AMD64 TFLite C++ prod tests"
description: "Testing DeepSpeech C++ for Linux/AMD64 on prod model, TFLite, optimized version"

13
taskcluster/test-nodejs_10x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,13 @@
build:
template_file: test-linux-opt-base.tyml
docker_image: "ubuntu:16.04"
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_xenial.prep_10} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 10.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 10.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v10.x on prod model, TFLite, optimized version"

13
taskcluster/test-nodejs_11x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,13 @@
build:
template_file: test-linux-opt-base.tyml
docker_image: "ubuntu:16.04"
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_xenial.prep_11} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 11.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 11.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v11.x on prod model, TFLite, optimized version"

13
taskcluster/test-nodejs_12x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,13 @@
build:
template_file: test-linux-opt-base.tyml
docker_image: "ubuntu:16.04"
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 12.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 12.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v12.x on prod model, TFLite, optimized version"

13
taskcluster/test-nodejs_13x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,13 @@
build:
template_file: test-linux-opt-base.tyml
docker_image: "ubuntu:16.04"
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_xenial.prep_13} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 13.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 13.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v13.x on prod model, TFLite, optimized version"

12
taskcluster/test-nodejs_4x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_trusty.prep_4} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 4.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 4.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v4.x on prod model, TFLite, optimized version"

12
taskcluster/test-nodejs_5x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_trusty.prep_5} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 5.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 5.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v5.x on prod model, TFLite, optimized version"

12
taskcluster/test-nodejs_6x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_trusty.prep_6} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 6.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 6.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v6.x on prod model, TFLite, optimized version"

12
taskcluster/test-nodejs_7x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_trusty.prep_7} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 7.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 7.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v7.x on prod model, TFLite, optimized version"

12
taskcluster/test-nodejs_8x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_trusty.prep_8} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 8.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 8.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v8.x on prod model, TFLite, optimized version"

12
taskcluster/test-nodejs_9x_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
${nodejs.packages_trusty.prep_9} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 9.x"
metadata:
name: "DeepSpeech Linux AMD64 TFLite NodeJS 9.x prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v9.x on prod model, TFLite, optimized version"

12
taskcluster/test-python_27m_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 2.7.14:m"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v2.7 m prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on Python v2.7 m on prod model, TFLite, optimized version"

12
taskcluster/test-python_27mu_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 2.7.14:mu"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v2.7 mu prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on Python v2.7 mu on prod model, TFLite, optimized version"

12
taskcluster/test-python_35_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.5.5:m"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v3.5 prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.5 on prod model, TFLite, optimized version"

12
taskcluster/test-python_36_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.6.4:m"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v3.6 prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.6 on prod model, TFLite, optimized version"

12
taskcluster/test-python_37_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.7.0:m"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v3.7 prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.7 on prod model, TFLite, optimized version"

12
taskcluster/test-python_38_tflite-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.8.0:"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v3.8 prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.8 on prod model, TFLite, optimized version"