From f80dbcda751ddf30b3df8c5477efd198becb2d27 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 24 Oct 2019 14:57:28 +0200 Subject: [PATCH] Build Python 3.8 wheels Fixes #2461 --- examples/mic_vad_streaming/requirements.txt | 4 +-- native_client/ctcdecode/Makefile | 4 +-- native_client/python/Makefile | 2 +- native_client/python/__init__.py | 14 ++++++--- native_client/swig_py38_untrack.patch | 26 ++++++++++++++++ taskcluster/.shared.yml | 4 +-- ...reaming-py38.yml.DISABLED_UNTIL_SCIPY_PY38 | 13 ++++++++ taskcluster/examples-vad_transcriber-py38.yml | 10 +++++++ taskcluster/tc-tests-utils.sh | 30 +++++++++++++++---- .../test-python_38-darwin-amd64-opt.yml | 13 ++++++++ .../test-python_38-linux-amd64-opt.yml | 13 ++++++++ ...python_38-linux-amd64-prod_pbmodel-opt.yml | 12 ++++++++ taskcluster/test-python_38-win-amd64-opt.yml | 13 ++++++++ taskcluster/win-build.sh | 2 +- 14 files changed, 143 insertions(+), 17 deletions(-) create mode 100644 native_client/swig_py38_untrack.patch create mode 100644 taskcluster/examples-mic_vad_streaming-py38.yml.DISABLED_UNTIL_SCIPY_PY38 create mode 100644 taskcluster/examples-vad_transcriber-py38.yml create mode 100644 taskcluster/test-python_38-darwin-amd64-opt.yml create mode 100644 taskcluster/test-python_38-linux-amd64-opt.yml create mode 100644 taskcluster/test-python_38-linux-amd64-prod_pbmodel-opt.yml create mode 100644 taskcluster/test-python_38-win-amd64-opt.yml diff --git a/examples/mic_vad_streaming/requirements.txt b/examples/mic_vad_streaming/requirements.txt index 4136ca09..76add2ba 100644 --- a/examples/mic_vad_streaming/requirements.txt +++ b/examples/mic_vad_streaming/requirements.txt @@ -2,5 +2,5 @@ deepspeech~=0.6.0a10 pyaudio~=0.2.11 webrtcvad~=2.0.10 halo~=0.0.18 -numpy~=1.15.1 -scipy~=1.1.0 +numpy>=1.15.1 +scipy>=1.1.0 diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index 98f8f9fd..23319e78 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -25,14 +25,14 @@ clean: clean-keep-common rm -f common.a bindings: clean-keep-common - pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0 + pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) find temp_build -type f -name "*.o" -delete AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build bindings-debug: clean-keep-common - pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0 + pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) $(GENERATE_DEBUG_SYMS) find temp_build -type f -name "*.o" -delete diff --git a/native_client/python/Makefile b/native_client/python/Makefile index 085ce8b0..1735fe77 100644 --- a/native_client/python/Makefile +++ b/native_client/python/Makefile @@ -7,7 +7,7 @@ bindings-clean: rm -f impl_wrap.cpp impl.py bindings-build: - pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0 + pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) MANIFEST.in: bindings-build diff --git a/native_client/python/__init__.py b/native_client/python/__init__.py index b9166632..5a3de5af 100644 --- a/native_client/python/__init__.py +++ b/native_client/python/__init__.py @@ -4,12 +4,18 @@ import platform #The API is not snake case which triggers linter errors #pylint: disable=invalid-name -# On Windows, we can't rely on RPATH being set to $ORIGIN/lib/ or on -# @loader_path/lib but we can change the PATH to include the proper directory -# for the dynamic linker if platform.system().lower() == "windows": dslib_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib') - os.environ['PATH'] = dslib_path + ';' + os.environ['PATH'] + + # On Windows, we can't rely on RPATH being set to $ORIGIN/lib/ or on + # @loader_path/lib + if hasattr(os, 'add_dll_directory'): + # Starting with Python 3.8 this properly handles the problem + os.add_dll_directory(dslib_path) + else: + # Before Pythin 3.8 we need to change the PATH to include the proper + # directory for the dynamic linker + os.environ['PATH'] = dslib_path + ';' + os.environ['PATH'] import deepspeech diff --git a/native_client/swig_py38_untrack.patch b/native_client/swig_py38_untrack.patch new file mode 100644 index 00000000..56ed4264 --- /dev/null +++ b/native_client/swig_py38_untrack.patch @@ -0,0 +1,26 @@ +From db9822788e183b79457d6bedab8b9a5cabb4cd5e Mon Sep 17 00:00:00 2001 +From: Christian Kellner +Date: Fri, 12 Jul 2019 18:33:29 +0200 +Subject: [PATCH] Use PyObject_GC_UnTrack in lieu of the old variant + +The _PyObject_GC_UNTRACK[1] macro got deprecated in 3.6 and finally +removed in 3.8. Therefore use PyObject_GC_UnTrack instead. + +[1] https://docs.python.org/3/c-api/gcsupport.html#c._PyObject_GC_UNTRACK +--- + Lib/python/builtin.swg | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Lib/python/builtin.swg b/Lib/python/builtin.swg +index 5062a8b424..28051e67b9 100644 +--- a/Lib/python/builtin.swg ++++ b/Lib/python/builtin.swg +@@ -117,7 +117,7 @@ SwigPyBuiltin_FunpackSetterClosure (PyObject *obj, PyObject *val, void *closure) + + SWIGINTERN void + SwigPyStaticVar_dealloc(PyDescrObject *descr) { +- _PyObject_GC_UNTRACK(descr); ++ PyObject_GC_UnTrack(descr); + Py_XDECREF(PyDescr_TYPE(descr)); + Py_XDECREF(PyDescr_NAME(descr)); + PyObject_GC_Del(descr); diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index c8692c57..cb9edec3 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -76,7 +76,7 @@ swig: packages: install_script: 'wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig_3.0.12-1_amd64.deb -O /tmp/swig_3.0.12-1_amd64.deb && wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig3.0_3.0.12-1_amd64.deb -O /tmp/swig3.0_3.0.12-1_amd64.deb && dpkg -i /tmp/swig_3.0.12-1_amd64.deb /tmp/swig3.0_3.0.12-1_amd64.deb' patch_nodejs: - linux: '(for patch_file in /home/build-user/DeepSpeech/ds/native_client/swig_node_v7x-v8x-v9x_*.patch /home/build-user/DeepSpeech/ds/native_client/swig_node_v10.12.patch /home/build-user/DeepSpeech/ds/native_client/swig_node_v12_*.patch ; do patch -d /usr/share/swig3.0/ -p2 < $patch_file; done)' + linux: '(for patch_file in /home/build-user/DeepSpeech/ds/native_client/swig_node_v7x-v8x-v9x_*.patch /home/build-user/DeepSpeech/ds/native_client/swig_node_v10.12.patch /home/build-user/DeepSpeech/ds/native_client/swig_node_v12_*.patch /home/build-user/DeepSpeech/ds/native_client/swig_py38_untrack.patch ; do patch -d /usr/share/swig3.0/ -p2 < $patch_file; done)' # Test if we can reverse patch #2: # - if yes, likely we are in a good state, patch _0001 and _0002 have # been applied @@ -87,7 +87,7 @@ swig: # applied, and we don't want to. So just test that reversing _0002 in # dry-run. osx_v12: '(if ! patch -R -d $TASKCLUSTER_TASK_DIR/homebrew/Cellar/swig/*/share/swig/*/ -p2 -s -f --dry-run < $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/swig_node_v12_0001-Upgrade-SWIG-to-support-NodeJS-v12-V8-v7.6-runtime.patch ; then -patch -d $TASKCLUSTER_TASK_DIR/homebrew/Cellar/swig/*/share/swig/*/ -p2 < $TASKCLUSTER_TASK_DIR/DeepSpeech/ds//native_client/swig_node_v12_0001-Upgrade-SWIG-to-support-NodeJS-v12-V8-v7.6-runtime.patch ; +patch -d $TASKCLUSTER_TASK_DIR/homebrew/Cellar/swig/*/share/swig/*/ -p2 < $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/swig_node_v12_0001-Upgrade-SWIG-to-support-NodeJS-v12-V8-v7.6-runtime.patch ; else echo "NO NODEJS v12 PATCH NEEDED"; fi)' diff --git a/taskcluster/examples-mic_vad_streaming-py38.yml.DISABLED_UNTIL_SCIPY_PY38 b/taskcluster/examples-mic_vad_streaming-py38.yml.DISABLED_UNTIL_SCIPY_PY38 new file mode 100644 index 00000000..0bd756ec --- /dev/null +++ b/taskcluster/examples-mic_vad_streaming-py38.yml.DISABLED_UNTIL_SCIPY_PY38 @@ -0,0 +1,13 @@ +build: + template_file: examples-base.tyml + docker_image: "python:3.8" + dependencies: + - "linux-amd64-cpu-opt" + system_setup: + > + apt-get -qq -y install portaudio19-dev pulseaudio + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/mic_vad_streaming/test.sh 3.8.0:" + metadata: + name: "DeepSpeech examples: mic VAD streaming Py3.8" + description: "DeepSpeech examples: mic VAD streaming Python 3.8" diff --git a/taskcluster/examples-vad_transcriber-py38.yml b/taskcluster/examples-vad_transcriber-py38.yml new file mode 100644 index 00000000..f0dfac7a --- /dev/null +++ b/taskcluster/examples-vad_transcriber-py38.yml @@ -0,0 +1,10 @@ +build: + template_file: examples-base.tyml + docker_image: "python:3.8" + dependencies: + - "linux-amd64-cpu-opt" + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/vad_transcriber/test.sh 3.8.0:" + metadata: + name: "DeepSpeech examples: VAD transcriber Py3.8" + description: "DeepSpeech examples: VAD transcriberaming Python 3.8" diff --git a/taskcluster/tc-tests-utils.sh b/taskcluster/tc-tests-utils.sh index ba477da8..e8edb75e 100755 --- a/taskcluster/tc-tests-utils.sh +++ b/taskcluster/tc-tests-utils.sh @@ -55,7 +55,7 @@ model_name="$(basename "${model_source}")" model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm" model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}" -SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.16:ucs2 2.7.16:ucs4 3.4.10:ucs4 3.5.7:ucs4 3.6.8:ucs4 3.7.3:ucs4} +SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.16:ucs2 2.7.16:ucs4 3.4.10:ucs4 3.5.7:ucs4 3.6.8:ucs4 3.7.3:ucs4 3.8.0:ucs4} SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-4.9.1 5.12.0 6.17.1 7.10.1 8.16.0 9.11.2 10.16.0 11.15.0 12.5.0} SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-1.6.18 1.7.16 1.8.8 2.0.18 3.0.16 3.1.11 4.0.3 4.1.5 4.2.5 5.0.6 6.0.11} @@ -604,13 +604,19 @@ install_pyenv() return; fi + # Allows updating local cache if required if [ ! -e "${PYENV_ROOT}/bin/pyenv" ]; then git clone --quiet https://github.com/pyenv/pyenv.git ${PYENV_ROOT} + else pushd ${PYENV_ROOT} - git checkout --quiet eb68ec9488f0df1f668e9272dd5bd8854edf1dff + git fetch origin popd fi + pushd ${PYENV_ROOT} + git checkout --quiet 0e7cfc3b3d4eca46ad83d632e1505f5932cd179b + popd + if [ ! -d "${PYENV_ROOT}/plugins/pyenv-alias" ]; then git clone https://github.com/s1341/pyenv-alias.git ${PYENV_ROOT}/plugins/pyenv-alias pushd ${PYENV_ROOT}/plugins/pyenv-alias @@ -982,7 +988,7 @@ maybe_ssl102_py37() ARCH=$(uname -m) case "${pyver}" in - 3.7*) + 3.7*|3.8*) if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial @@ -1010,8 +1016,16 @@ maybe_ssl102_py37() export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/" fi; - export NUMPY_BUILD_VERSION="==1.14.5" - export NUMPY_DEP_VERSION=">=1.14.5" + case "${pyver}" in + 3.7*) + export NUMPY_BUILD_VERSION="==1.14.5" + export NUMPY_DEP_VERSION=">=1.14.5" + ;; + 3.8*) + export NUMPY_BUILD_VERSION="==1.17.3" + export NUMPY_DEP_VERSION=">=1.17.3" + ;; + esac ;; esac } @@ -1039,6 +1053,10 @@ maybe_numpy_min_version_winamd64() export NUMPY_BUILD_VERSION="==1.14.5" export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0" ;; + 3.8*) + export NUMPY_BUILD_VERSION="==1.17.3" + export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3" + ;; esac } @@ -1086,6 +1104,8 @@ extract_python_versions() local _pyconf="ucs2" elif [ "${_py_unicode_type}" = "mu" ]; then local _pyconf="ucs4" + elif [ "${_py_unicode_type}" = "" ]; then # valid for Python 3.8 + local _pyconf="ucs4" fi; local _pyalias="${_pyver}_${_pyconf}" diff --git a/taskcluster/test-python_38-darwin-amd64-opt.yml b/taskcluster/test-python_38-darwin-amd64-opt.yml new file mode 100644 index 00000000..bf966765 --- /dev/null +++ b/taskcluster/test-python_38-darwin-amd64-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-darwin-opt-base.tyml + dependencies: + - "darwin-amd64-cpu-opt" + - "test-training_upstream-linux-amd64-py35m-opt" + system_setup: + > + ${python.brew.setup} && ${python.brew.env} + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.0:" + metadata: + name: "DeepSpeech OSX AMD64 CPU Python v3.8 tests" + description: "Testing DeepSpeech for OSX/AMD64 on Python v3.8.0:m, CPU only, optimized version" diff --git a/taskcluster/test-python_38-linux-amd64-opt.yml b/taskcluster/test-python_38-linux-amd64-opt.yml new file mode 100644 index 00000000..a213afc0 --- /dev/null +++ b/taskcluster/test-python_38-linux-amd64-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-cpu-opt" + - "test-training_upstream-linux-amd64-py35m-opt" + system_setup: + > + apt-get -qq -y install ${python.packages_trusty.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.0:" + metadata: + name: "DeepSpeech Linux AMD64 CPU Python v3.8 tests" + description: "Testing DeepSpeech for Linux/AMD64 on Python v3.8, CPU only, optimized version" diff --git a/taskcluster/test-python_38-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_38-linux-amd64-prod_pbmodel-opt.yml new file mode 100644 index 00000000..1b767056 --- /dev/null +++ b/taskcluster/test-python_38-linux-amd64-prod_pbmodel-opt.yml @@ -0,0 +1,12 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-cpu-opt" + system_setup: + > + apt-get -qq -y install ${python.packages_trusty.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.8.0:" + metadata: + name: "DeepSpeech Linux AMD64 CPU Python v3.8 prod tests" + description: "Testing DeepSpeech for Linux/AMD64 on Python v3.8 on prod model, CPU only, optimized version" diff --git a/taskcluster/test-python_38-win-amd64-opt.yml b/taskcluster/test-python_38-win-amd64-opt.yml new file mode 100644 index 00000000..0d03c406 --- /dev/null +++ b/taskcluster/test-python_38-win-amd64-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-cpu-opt" + - "test-training_upstream-linux-amd64-py35m-opt" + system_setup: + > + ${system.sox_win} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.0:" + metadata: + name: "DeepSpeech Windows AMD64 CPU Python v3.8 tests" + description: "Testing DeepSpeech for Windows/AMD64 on Python v3.8, CPU only, optimized version" diff --git a/taskcluster/win-build.sh b/taskcluster/win-build.sh index f6ef61ee..a0ce9d90 100755 --- a/taskcluster/win-build.sh +++ b/taskcluster/win-build.sh @@ -40,7 +40,7 @@ export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin:'/c/Program Files/nodejs/' do_deepspeech_binary_build # Those are the versions available on NuGet.org -export SUPPORTED_PYTHON_VERSIONS="3.5.4 3.6.7 3.7.1" +export SUPPORTED_PYTHON_VERSIONS="3.5.4 3.6.7 3.7.1 3.8.0" do_deepspeech_python_build "${cuda}" do_deepspeech_nodejs_build "${cuda}"