experiments
/
STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

initial commit for py39 support

This commit is contained in:
dag7dev 2020-11-04 15:52:43 +01:00 committed by Alexandre Lissy
parent b72e2643c4
commit 3a2879933f
26 changed files with 224 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
native_client/ctcdecode/Makefile
View File

@ -44,14 +44,14 @@ workspace_status.cc:
# Enforce PATH here because swig calls from build_ext looses track of some
# variables over several runs
bindings: clean-keep-third-party workspace_status.cc ds-swig
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
find temp_build -type f -name "*.o" -delete
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
rm -rf temp_build
bindings-debug: clean-keep-third-party workspace_status.cc ds-swig
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
$(GENERATE_DEBUG_SYMS)
find temp_build -type f -name "*.o" -delete

2
native_client/python/Makefile
View File

@ -9,7 +9,7 @@ bindings-clean:
# Enforce PATH here because swig calls from build_ext looses track of some
# variables over several runs
bindings-build: ds-swig
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0
DISTUTILS_USE_SDK=1 PATH=$(TOOLCHAIN):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
MANIFEST.in: bindings-build

12
taskcluster/.shared.yml
View File

@ -137,14 +137,14 @@ system:
namespace: 'project.deepspeech.gradle.7'
pyenv:
linux:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.9/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.linux.9'
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.18/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.linux.18'
osx:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.osx.9/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.osx.9'
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.osx.18/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.osx.18'
win:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.8/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.win.8'
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.18/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.win.18'
kenlm:
android_arm64_cpu:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.kenlm.689a25aae9171b3ea46bd80d4189f540f35f1a02.4.android-arm64/artifacts/public/kenlm.tar.gz"

1
taskcluster/examples-mic_vad_streaming-py38.yml.DISABLED_UNTIL_SCIPY_PY38 → taskcluster/examples-mic_vad_streaming-py38.yml
View File

@ -8,6 +8,7 @@ build:
apt-get -qq -y install portaudio19-dev pulseaudio
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/examples/mic_vad_streaming/test.sh 3.8.0:"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech examples: mic VAD streaming Py3.8"
description: "DeepSpeech examples: mic VAD streaming Python 3.8"

14
taskcluster/examples-mic_vad_streaming-py39.yml.DISABLED_UNTIL_SCIPY_PY39 Normal file
View File

@ -0,0 +1,14 @@
build:
template_file: examples-base.tyml
docker_image: "python:3.9"
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install portaudio19-dev pulseaudio
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/examples/mic_vad_streaming/test.sh 3.9.0:"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech examples: mic VAD streaming Py3.9"
description: "DeepSpeech examples: mic VAD streaming Python 3.9"

2
taskcluster/examples-vad_transcriber-py35.yml
View File

@ -8,4 +8,4 @@ build:
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech examples: VAD transcriber Py3.5"
description: "DeepSpeech examples: VAD transcriberaming Python 3.5"
description: "DeepSpeech examples: VAD transcriber streaming Python 3.5"

2
taskcluster/examples-vad_transcriber-py36.yml
View File

@ -8,4 +8,4 @@ build:
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech examples: VAD transcriber Py3.6"
description: "DeepSpeech examples: VAD transcriberaming Python 3.6"
description: "DeepSpeech examples: VAD transcriber streaming Python 3.6"

2
taskcluster/examples-vad_transcriber-py37.yml
View File

@ -8,4 +8,4 @@ build:
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech examples: VAD transcriber Py3.7"
description: "DeepSpeech examples: VAD transcriberaming Python 3.7"
description: "DeepSpeech examples: VAD transcriber streaming Python 3.7"

2
taskcluster/examples-vad_transcriber-py38.yml
View File

@ -8,4 +8,4 @@ build:
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech examples: VAD transcriber Py3.8"
description: "DeepSpeech examples: VAD transcriberaming Python 3.8"
description: "DeepSpeech examples: VAD transcriber streaming Python 3.8"

11
taskcluster/examples-vad_transcriber-py39.yml Normal file
View File

@ -0,0 +1,11 @@
build:
template_file: examples-base.tyml
docker_image: "python:3.9"
dependencies:
- "linux-amd64-cpu-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/examples/vad_transcriber/test.sh 3.9.0:"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech examples: VAD transcriber Py3.9"
description: "DeepSpeech examples: VAD transcriber streaming Python 3.9"

4
taskcluster/tc-all-vars.sh
View File

@ -16,7 +16,7 @@ if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
export DS_CPU_COUNT=$(nproc)
# Those are the versions available on NuGet.org
export SUPPORTED_PYTHON_VERSIONS="3.5.4:ucs2 3.6.8:ucs2 3.7.6:ucs2 3.8.1:ucs2"
export SUPPORTED_PYTHON_VERSIONS="3.5.4:ucs2 3.6.8:ucs2 3.7.6:ucs2 3.8.1:ucs2 3.9.0:ucs2"
fi;
if [ "${OS}" = "Darwin" ]; then
@ -79,7 +79,7 @@ model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}"
ldc93s1_sample_filename=''
SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-3.5.8:ucs2 3.6.10:ucs2 3.7.6:ucs2 3.8.1:ucs2}
SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-3.5.8:ucs2 3.6.10:ucs2 3.7.6:ucs2 3.8.1:ucs2 3.9.0:ucs2}
# When updating NodeJS / ElectronJS supported versions, do not forget to increment
# deepspeech.node-gyp-cache.<X> in both `system.node_gyp_cache` (taskcluster/.shared.yml)

18
taskcluster/tc-py-utils.sh
View File

@ -24,7 +24,7 @@ install_pyenv()
fi
pushd ${PYENV_ROOT}
git checkout --quiet 20a1f0cd7a3d2f95800d8e0d5863b4e98f25f4df
git checkout --quiet 806b30d6ce5b263a765648fbcdd68266833b7289
popd
if [ ! -d "${PYENV_ROOT}/plugins/pyenv-alias" ]; then
@ -213,6 +213,10 @@ maybe_numpy_min_version()
export NUMPY_BUILD_VERSION="==1.17.3"
export NUMPY_DEP_VERSION=">=1.17.3"
;;
3.9*)
export NUMPY_BUILD_VERSION="==1.19.4"
export NUMPY_DEP_VERSION=">=1.19.4"
;;
esac
;;
@ -230,6 +234,10 @@ maybe_numpy_min_version()
export NUMPY_BUILD_VERSION="==1.17.3"
export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3"
;;
3.9*)
export NUMPY_BUILD_VERSION="==1.19.4"
export NUMPY_DEP_VERSION=">=1.19.4"
;;
esac
;;
@ -251,6 +259,10 @@ maybe_numpy_min_version()
export NUMPY_BUILD_VERSION="==1.17.3"
export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3"
;;
3.9*)
export NUMPY_BUILD_VERSION="==1.19.4"
export NUMPY_DEP_VERSION=">=1.19.4"
;;
esac
;;
@ -310,7 +322,7 @@ extract_python_versions()
local _pyver=$(echo "${_pyver_full}" | cut -d':' -f1)
# 3.8.x => 38
# 3.8.x => 38 / 3.9.x => 39
local _pyver_pkg=$(echo "${_pyver}" | cut -d'.' -f1,2 | tr -d '.')
# https://www.python.org/dev/peps/pep-3149/#proposal
@ -321,7 +333,7 @@ extract_python_versions()
local _pyconf="ucs2"
elif [ "${_py_unicode_type}" = "mu" ]; then
local _pyconf="ucs4"
elif [ "${_py_unicode_type}" = "" ]; then # valid for Python 3.8
elif [ "${_py_unicode_type}" = "" ]; then # valid for Python 3.8 and 3.9
local _pyconf="ucs2"
fi;

12
taskcluster/test-python_39-darwin-amd64-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-darwin-opt-base.tyml
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.9.0: 16k"
metadata:
name: "DeepSpeech OSX AMD64 CPU Python v3.9 tests"
description: "Testing DeepSpeech for OSX/AMD64 on Python v3.9.0:m, CPU only, optimized version"

14
taskcluster/test-python_39-win-amd64-opt.yml Normal file
View File

@ -0,0 +1,14 @@
build:
template_file: test-win-opt-base.tyml
dependencies:
- "win-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${system.sox_win}
args:
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.9.0: 16k"
metadata:
name: "DeepSpeech Windows AMD64 CPU Python v3.9 tests"
description: "Testing DeepSpeech for Windows/AMD64 on Python v3.9, CPU only, optimized version"

14
taskcluster/test-python_39-win-cuda-opt.yml Normal file
View File

@ -0,0 +1,14 @@
build:
template_file: test-win-cuda-opt-base.tyml
dependencies:
- "win-amd64-gpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${system.sox_win}
args:
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.9.0: 16k cuda"
metadata:
name: "DeepSpeech Windows AMD64 CUDA Python v3.9 tests"
description: "Testing DeepSpeech for Windows/AMD64 on Python v3.9, CUDA, optimized version"

12
taskcluster/test-python_39_16k-linux-amd64-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.9.0: 16k"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech Linux AMD64 CPU Python v3.9 tests (16kHz)"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.9, CPU only, optimized version (16kHz)"

10
taskcluster/test-python_39_16k-linux-amd64-prod_pbmodel-opt.yml Normal file
View File

@ -0,0 +1,10 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.9.0: 16k"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech Linux AMD64 CPU Python v3.9 prod tests (16kHz)"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.9 on prod model, CPU only, optimized version (16kHz)"

12
taskcluster/test-python_39_8k-linux-amd64-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
- "test-training_8k-linux-amd64-py36m-opt"
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.9.0: 8k"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech Linux AMD64 CPU Python v3.9 tests (8kHz)"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.9, CPU only, optimized version (8kHz)"

10
taskcluster/test-python_39_8k-linux-amd64-prod_pbmodel-opt.yml Normal file
View File

@ -0,0 +1,10 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.9.0: 8k"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech Linux AMD64 CPU Python v3.9 prod tests (8kHz)"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.9 on prod model, CPU only, optimized version (8kHz)"

12
taskcluster/test-python_39_tflite_16k-darwin-amd64-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-darwin-opt-base.tyml
dependencies:
- "darwin-amd64-tflite-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python_tflite-tests.sh 3.9.0: 16k"
metadata:
name: "DeepSpeech OSX AMD64 TFLite Python v3.9 tests (16kHz)"
description: "Testing DeepSpeech for OSX/AMD64 on Python v3.9 TFLite, optimized version (16kHz)"

12
taskcluster/test-python_39_tflite_16k-darwin-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-darwin-opt-base.tyml
dependencies:
- "darwin-amd64-tflite-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.9.0: 16k"
metadata:
name: "DeepSpeech OSX AMD64 TFLite Python v3.9 prod tests (16kHz)"
description: "Testing DeepSpeech for OSX/AMD64 on Python v3.9 on prod model, TFLite, optimized version (16kHz)"

12
taskcluster/test-python_39_tflite_16k-linux-amd64-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests.sh 3.9.0: 16k"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v3.9 tests (16kHz)"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.9 TFLite, optimized version (16kHz)"

10
taskcluster/test-python_39_tflite_16k-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,10 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.9.0: 16k"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v3.9 prod tests (16kHz)"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.9 on prod model, TFLite, optimized version (16kHz)"

14
taskcluster/test-python_39_tflite_16k-win-amd64-opt.yml Normal file
View File

@ -0,0 +1,14 @@
build:
template_file: test-win-opt-base.tyml
dependencies:
- "win-amd64-tflite-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${system.sox_win}
args:
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests.sh 3.9.0: 16k"
metadata:
name: "DeepSpeech Windows AMD64 TFLite Python v3.9 tests (16kHz)"
description: "Testing DeepSpeech for Windows/AMD64 on Python v3.9 TFLite, optimized version (16kHz)"

14
taskcluster/test-python_39_tflite_16k-win-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,14 @@
build:
template_file: test-win-opt-base.tyml
dependencies:
- "win-amd64-tflite-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${system.sox_win}
args:
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.9.0: 16k"
metadata:
name: "DeepSpeech Windows AMD64 TFLite Python v3.9 prod tests (16kHz)"
description: "Testing DeepSpeech for Windows/AMD64 on Python v3.9 on prod model, TFLite, optimized version (16kHz)"

10
taskcluster/test-python_39_tflite_8k-linux-amd64-prod-opt.yml Normal file
View File

@ -0,0 +1,10 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.9.0: 8k"
workerType: "${docker.dsTests}"
metadata:
name: "DeepSpeech Linux AMD64 TFLite Python v3.9 prod tests (8kHz)"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.9 on prod model, TFLite, optimized version (8kHz)"