experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Build Python 3.8 wheels

Browse Source 
Fixes #2461
This commit is contained in:
Alexandre Lissy 2019-10-24 14:57:28 +02:00
parent bde5c31fd4
commit f80dbcda75
14 changed files with 143 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/mic_vad_streaming/requirements.txt
View File

@ -2,5 +2,5 @@ deepspeech~=0.6.0a10
pyaudio~=0.2.11
webrtcvad~=2.0.10
halo~=0.0.18
numpy~=1.15.1
scipy~=1.1.0
numpy>=1.15.1
scipy>=1.1.0

4
native_client/ctcdecode/Makefile
View File

@ -25,14 +25,14 @@ clean: clean-keep-common
rm -f common.a
bindings: clean-keep-common
pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
find temp_build -type f -name "*.o" -delete
AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
rm -rf temp_build
bindings-debug: clean-keep-common
pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
$(GENERATE_DEBUG_SYMS)
find temp_build -type f -name "*.o" -delete

2
native_client/python/Makefile
View File

@ -7,7 +7,7 @@ bindings-clean:
rm -f impl_wrap.cpp impl.py
bindings-build:
pip install --quiet $(PYTHON_PACKAGES) wheel==0.31.0 setuptools==39.1.0
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
PATH=$(TOOLCHAIN):$$PATH AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
MANIFEST.in: bindings-build

14
native_client/python/__init__.py
View File

@ -4,12 +4,18 @@ import platform
#The API is not snake case which triggers linter errors
#pylint: disable=invalid-name
# On Windows, we can't rely on RPATH being set to $ORIGIN/lib/ or on
# @loader_path/lib but we can change the PATH to include the proper directory
# for the dynamic linker
if platform.system().lower() == "windows":
dslib_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')
os.environ['PATH'] = dslib_path + ';' + os.environ['PATH']
# On Windows, we can't rely on RPATH being set to $ORIGIN/lib/ or on
# @loader_path/lib
if hasattr(os, 'add_dll_directory'):
# Starting with Python 3.8 this properly handles the problem
os.add_dll_directory(dslib_path)
else:
# Before Pythin 3.8 we need to change the PATH to include the proper
# directory for the dynamic linker
os.environ['PATH'] = dslib_path + ';' + os.environ['PATH']
import deepspeech

26
native_client/swig_py38_untrack.patch Normal file
View File

@ -0,0 +1,26 @@
From db9822788e183b79457d6bedab8b9a5cabb4cd5e Mon Sep 17 00:00:00 2001
From: Christian Kellner <christian@kellner.me>
Date: Fri, 12 Jul 2019 18:33:29 +0200
Subject: [PATCH] Use PyObject_GC_UnTrack in lieu of the old variant
The _PyObject_GC_UNTRACK[1] macro got deprecated in 3.6 and finally
removed in 3.8. Therefore use PyObject_GC_UnTrack instead.
[1] https://docs.python.org/3/c-api/gcsupport.html#c._PyObject_GC_UNTRACK
---
Lib/python/builtin.swg | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Lib/python/builtin.swg b/Lib/python/builtin.swg
index 5062a8b424..28051e67b9 100644
--- a/Lib/python/builtin.swg
+++ b/Lib/python/builtin.swg
@@ -117,7 +117,7 @@ SwigPyBuiltin_FunpackSetterClosure (PyObject *obj, PyObject *val, void *closure)
SWIGINTERN void
SwigPyStaticVar_dealloc(PyDescrObject *descr) {
- _PyObject_GC_UNTRACK(descr);
+ PyObject_GC_UnTrack(descr);
Py_XDECREF(PyDescr_TYPE(descr));
Py_XDECREF(PyDescr_NAME(descr));
PyObject_GC_Del(descr);

4
taskcluster/.shared.yml
View File

@ -76,7 +76,7 @@ swig:
packages:
install_script: 'wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig_3.0.12-1_amd64.deb -O /tmp/swig_3.0.12-1_amd64.deb && wget http://mirrors.kernel.org/ubuntu/pool/universe/s/swig/swig3.0_3.0.12-1_amd64.deb -O /tmp/swig3.0_3.0.12-1_amd64.deb && dpkg -i /tmp/swig_3.0.12-1_amd64.deb /tmp/swig3.0_3.0.12-1_amd64.deb'
patch_nodejs:
linux: '(for patch_file in /home/build-user/DeepSpeech/ds/native_client/swig_node_v7x-v8x-v9x_*.patch /home/build-user/DeepSpeech/ds/native_client/swig_node_v10.12.patch /home/build-user/DeepSpeech/ds/native_client/swig_node_v12_*.patch ; do patch -d /usr/share/swig3.0/ -p2 < $patch_file; done)'
linux: '(for patch_file in /home/build-user/DeepSpeech/ds/native_client/swig_node_v7x-v8x-v9x_*.patch /home/build-user/DeepSpeech/ds/native_client/swig_node_v10.12.patch /home/build-user/DeepSpeech/ds/native_client/swig_node_v12_*.patch /home/build-user/DeepSpeech/ds/native_client/swig_py38_untrack.patch ; do patch -d /usr/share/swig3.0/ -p2 < $patch_file; done)'
# Test if we can reverse patch #2:
# - if yes, likely we are in a good state, patch _0001 and _0002 have
# been applied
@ -87,7 +87,7 @@ swig:
# applied, and we don't want to. So just test that reversing _0002 in
# dry-run.
osx_v12: '(if ! patch -R -d $TASKCLUSTER_TASK_DIR/homebrew/Cellar/swig/*/share/swig/*/ -p2 -s -f --dry-run < $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/swig_node_v12_0001-Upgrade-SWIG-to-support-NodeJS-v12-V8-v7.6-runtime.patch ; then
patch -d $TASKCLUSTER_TASK_DIR/homebrew/Cellar/swig/*/share/swig/*/ -p2 < $TASKCLUSTER_TASK_DIR/DeepSpeech/ds//native_client/swig_node_v12_0001-Upgrade-SWIG-to-support-NodeJS-v12-V8-v7.6-runtime.patch ;
patch -d $TASKCLUSTER_TASK_DIR/homebrew/Cellar/swig/*/share/swig/*/ -p2 < $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/swig_node_v12_0001-Upgrade-SWIG-to-support-NodeJS-v12-V8-v7.6-runtime.patch ;
else
echo "NO NODEJS v12 PATCH NEEDED";
fi)'

13
taskcluster/examples-mic_vad_streaming-py38.yml.DISABLED_UNTIL_SCIPY_PY38 Normal file
View File

@ -0,0 +1,13 @@
build:
template_file: examples-base.tyml
docker_image: "python:3.8"
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install portaudio19-dev pulseaudio
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/mic_vad_streaming/test.sh 3.8.0:"
metadata:
name: "DeepSpeech examples: mic VAD streaming Py3.8"
description: "DeepSpeech examples: mic VAD streaming Python 3.8"

10
taskcluster/examples-vad_transcriber-py38.yml Normal file
View File

@ -0,0 +1,10 @@
build:
template_file: examples-base.tyml
docker_image: "python:3.8"
dependencies:
- "linux-amd64-cpu-opt"
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/examples/vad_transcriber/test.sh 3.8.0:"
metadata:
name: "DeepSpeech examples: VAD transcriber Py3.8"
description: "DeepSpeech examples: VAD transcriberaming Python 3.8"

30
taskcluster/tc-tests-utils.sh
View File

@ -55,7 +55,7 @@ model_name="$(basename "${model_source}")"
model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm"
model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}"
SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.16:ucs2 2.7.16:ucs4 3.4.10:ucs4 3.5.7:ucs4 3.6.8:ucs4 3.7.3:ucs4}
SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.16:ucs2 2.7.16:ucs4 3.4.10:ucs4 3.5.7:ucs4 3.6.8:ucs4 3.7.3:ucs4 3.8.0:ucs4}
SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-4.9.1 5.12.0 6.17.1 7.10.1 8.16.0 9.11.2 10.16.0 11.15.0 12.5.0}
SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-1.6.18 1.7.16 1.8.8 2.0.18 3.0.16 3.1.11 4.0.3 4.1.5 4.2.5 5.0.6 6.0.11}
@ -604,13 +604,19 @@ install_pyenv()
return;
fi
# Allows updating local cache if required
if [ ! -e "${PYENV_ROOT}/bin/pyenv" ]; then
git clone --quiet https://github.com/pyenv/pyenv.git ${PYENV_ROOT}
else
pushd ${PYENV_ROOT}
git checkout --quiet eb68ec9488f0df1f668e9272dd5bd8854edf1dff
git fetch origin
popd
fi
pushd ${PYENV_ROOT}
git checkout --quiet 0e7cfc3b3d4eca46ad83d632e1505f5932cd179b
popd
if [ ! -d "${PYENV_ROOT}/plugins/pyenv-alias" ]; then
git clone https://github.com/s1341/pyenv-alias.git ${PYENV_ROOT}/plugins/pyenv-alias
pushd ${PYENV_ROOT}/plugins/pyenv-alias
@ -982,7 +988,7 @@ maybe_ssl102_py37()
ARCH=$(uname -m)
case "${pyver}" in
3.7*)
3.7*|3.8*)
if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then
PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial
@ -1010,8 +1016,16 @@ maybe_ssl102_py37()
export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/"
fi;
export NUMPY_BUILD_VERSION="==1.14.5"
export NUMPY_DEP_VERSION=">=1.14.5"
case "${pyver}" in
3.7*)
export NUMPY_BUILD_VERSION="==1.14.5"
export NUMPY_DEP_VERSION=">=1.14.5"
;;
3.8*)
export NUMPY_BUILD_VERSION="==1.17.3"
export NUMPY_DEP_VERSION=">=1.17.3"
;;
esac
;;
esac
}
@ -1039,6 +1053,10 @@ maybe_numpy_min_version_winamd64()
export NUMPY_BUILD_VERSION="==1.14.5"
export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0"
;;
3.8*)
export NUMPY_BUILD_VERSION="==1.17.3"
export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3"
;;
esac
}
@ -1086,6 +1104,8 @@ extract_python_versions()
local _pyconf="ucs2"
elif [ "${_py_unicode_type}" = "mu" ]; then
local _pyconf="ucs4"
elif [ "${_py_unicode_type}" = "" ]; then # valid for Python 3.8
local _pyconf="ucs4"
fi;
local _pyalias="${_pyver}_${_pyconf}"

13
taskcluster/test-python_38-darwin-amd64-opt.yml Normal file
View File

@ -0,0 +1,13 @@
build:
template_file: test-darwin-opt-base.tyml
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_upstream-linux-amd64-py35m-opt"
system_setup:
>
${python.brew.setup} && ${python.brew.env}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.0:"
metadata:
name: "DeepSpeech OSX AMD64 CPU Python v3.8 tests"
description: "Testing DeepSpeech for OSX/AMD64 on Python v3.8.0:m, CPU only, optimized version"

13
taskcluster/test-python_38-linux-amd64-opt.yml Normal file
View File

@ -0,0 +1,13 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
- "test-training_upstream-linux-amd64-py35m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.0:"
metadata:
name: "DeepSpeech Linux AMD64 CPU Python v3.8 tests"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.8, CPU only, optimized version"

12
taskcluster/test-python_38-linux-amd64-prod_pbmodel-opt.yml Normal file
View File

@ -0,0 +1,12 @@
build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.8.0:"
metadata:
name: "DeepSpeech Linux AMD64 CPU Python v3.8 prod tests"
description: "Testing DeepSpeech for Linux/AMD64 on Python v3.8 on prod model, CPU only, optimized version"

13
taskcluster/test-python_38-win-amd64-opt.yml Normal file
View File

@ -0,0 +1,13 @@
build:
template_file: test-win-opt-base.tyml
dependencies:
- "win-amd64-cpu-opt"
- "test-training_upstream-linux-amd64-py35m-opt"
system_setup:
>
${system.sox_win}
args:
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.0:"
metadata:
name: "DeepSpeech Windows AMD64 CPU Python v3.8 tests"
description: "Testing DeepSpeech for Windows/AMD64 on Python v3.8, CPU only, optimized version"

2
taskcluster/win-build.sh
View File

@ -40,7 +40,7 @@ export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin:'/c/Program Files/nodejs/'
do_deepspeech_binary_build
# Those are the versions available on NuGet.org
export SUPPORTED_PYTHON_VERSIONS="3.5.4 3.6.7 3.7.1"
export SUPPORTED_PYTHON_VERSIONS="3.5.4 3.6.7 3.7.1 3.8.0"
do_deepspeech_python_build "${cuda}"
do_deepspeech_nodejs_build "${cuda}"