From 6dbf65cce8f8b160b55a9ea62564b714170f5b5d Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Wed, 3 Apr 2019 16:27:07 +0200 Subject: [PATCH] Produce Windows Python bindings Fixes #1937 --- native_client/definitions.mk | 2 + native_client/python/Makefile | 4 +- native_client/python/__init__.py | 10 + taskcluster/.shared.yml | 2 +- taskcluster/test-nodejs_10x-win-amd64-opt.yml | 2 +- taskcluster/test-nodejs_11x-win-amd64-opt.yml | 2 +- taskcluster/test-nodejs_4x-win-amd64-opt.yml | 2 +- taskcluster/test-nodejs_6x-win-amd64-opt.yml | 2 +- taskcluster/test-nodejs_7x-win-amd64-opt.yml | 2 +- taskcluster/test-nodejs_8x-win-amd64-opt.yml | 2 +- taskcluster/test-nodejs_9x-win-amd64-opt.yml | 2 +- taskcluster/test-python_35-win-amd64-opt.yml | 13 ++ taskcluster/test-python_36-win-amd64-opt.yml | 13 ++ taskcluster/test-python_37-win-amd64-opt.yml | 13 ++ taskcluster/win-build.sh | 4 + taskcluster/win-package.sh | 4 + tc-python-tests-prod.sh | 36 +--- tc-python-tests.sh | 38 ++-- tc-single-shot-inference.sh | 37 +--- tc-tests-utils.sh | 180 +++++++++++++++++- 20 files changed, 271 insertions(+), 99 deletions(-) create mode 100644 taskcluster/test-python_35-win-amd64-opt.yml create mode 100644 taskcluster/test-python_36-win-amd64-opt.yml create mode 100644 taskcluster/test-python_37-win-amd64-opt.yml diff --git a/native_client/definitions.mk b/native_client/definitions.mk index b9116152..da404a69 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -40,6 +40,7 @@ LINK_PATH_DEEPSPEECH := CFLAGS_DEEPSPEECH := -nologo -Fe$(DEEPSPEECH_BIN) SOX_CFLAGS := SOX_LDFLAGS := +PYTHON_PACKAGES := numpy${NUMPY_BUILD_VERSION} endif ifeq ($(TARGET),rpi3) @@ -144,6 +145,7 @@ define copy_missing_libs \ for missing in $$missing_libs; do \ find $(SO_SEARCH) -type f -name "$$missing" -exec cp {} $$TARGET_LIB_DIR \; ; \ + chmod +w $$TARGET_LIB_DIR/*.so ; \ if [ ! -z "$$MANIFEST_IN" ]; then \ echo "include $$TARGET_LIB_DIR/$$missing" >> $$MANIFEST_IN; \ fi; \ diff --git a/native_client/python/Makefile b/native_client/python/Makefile index 9758baf0..d03de4c0 100644 --- a/native_client/python/Makefile +++ b/native_client/python/Makefile @@ -14,9 +14,11 @@ MANIFEST.in: bindings-build > $@ $(call copy_missing_libs,temp_build/deepspeech/*.so,temp_build/deepspeech/lib,$@) +# On Unix, _wrap.o gets generated +# On Windows, _wrap.obj it is bindings-package: MANIFEST.in cat MANIFEST.in - rm temp_build/*_wrap.o + rm -f temp_build/*_wrap.o temp_build/Release/*_wrap.obj AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) bindings: bindings-build bindings-package diff --git a/native_client/python/__init__.py b/native_client/python/__init__.py index a24dd720..a68fa439 100644 --- a/native_client/python/__init__.py +++ b/native_client/python/__init__.py @@ -1,3 +1,13 @@ +import os +import platform + +# On Windows, we can't rely on RPATH being set to $ORIGIN/lib/ or on +# @loader_path/lib but we can change the PATH to include the proper directory +# for the dynamic linker +if platform.system().lower() == "windows": + dslib_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib') + os.environ['PATH'] = dslib_path + ';' + os.environ['PATH'] + import deepspeech # rename for backwards compatibility diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 53f1682b..1ec0a73c 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -50,7 +50,6 @@ nodejs: prep_10: 'nvm install 10.12.0 && nvm use 10.12.0' prep_11: 'nvm install 11.0.0 && nvm use 11.0.0' win: - sox: '/usr/bin/wget.exe https://sourceforge.net/projects/sox/files/sox/14.4.2/sox-14.4.2-win32.zip/download -O sox-14.4.2-win32.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_TASK_DIR/bin/ -tzip -aoa sox-14.4.2-win32.zip && rm sox-*zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/sox-14.4.2/:$PATH' prep_4: '/usr/bin/wget.exe https://nodejs.org/dist/v4.9.1/node-v4.9.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v4.9.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v4.9.1-win-x64/:$PATH' prep_6: '/usr/bin/wget.exe https://nodejs.org/dist/v6.14.4/node-v6.14.4-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v6.14.4-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v6.14.4-win-x64/:$PATH' prep_7: '/usr/bin/wget.exe https://nodejs.org/dist/v7.10.1/node-v7.10.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v7.10.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v7.10.1-win-x64/:$PATH' @@ -91,6 +90,7 @@ system: linux: '/home/build-user' osx: '/Users/build-user' win: '/c/builds/tc-workdir' + sox_win: '/usr/bin/wget.exe https://sourceforge.net/projects/sox/files/sox/14.4.2/sox-14.4.2-win32.zip/download -O sox-14.4.2-win32.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_TASK_DIR/bin/ -tzip -aoa sox-14.4.2-win32.zip && rm sox-*zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/sox-14.4.2/:$PATH' notifications: irc: '#machinelearning' aptEc2Mirrors: 'echo "deb http://archive.ubuntu.com/ubuntu/ trusty-updates main" > /etc/apt/sources.list.d/trusty-updates.list && sed -ri -e "s|archive.ubuntu.com|${TASKCLUSTER_WORKER_GROUP}.ec2.archive.ubuntu.com|g" -e "s|security.ubuntu.com|${TASKCLUSTER_WORKER_GROUP}.ec2.archive.ubuntu.com|g" /etc/apt/sources.list && apt-get -qq update && apt-get -qq -y upgrade' diff --git a/taskcluster/test-nodejs_10x-win-amd64-opt.yml b/taskcluster/test-nodejs_10x-win-amd64-opt.yml index 5a517390..92d4586e 100644 --- a/taskcluster/test-nodejs_10x-win-amd64-opt.yml +++ b/taskcluster/test-nodejs_10x-win-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.win.sox} && ${nodejs.win.prep_10} + ${system.sox_win} && ${nodejs.win.prep_10} args: tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-node-tests.sh 10.x" metadata: diff --git a/taskcluster/test-nodejs_11x-win-amd64-opt.yml b/taskcluster/test-nodejs_11x-win-amd64-opt.yml index 2a3c28ce..3aae86cf 100644 --- a/taskcluster/test-nodejs_11x-win-amd64-opt.yml +++ b/taskcluster/test-nodejs_11x-win-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.win.sox} && ${nodejs.win.prep_11} + ${system.sox_win} && ${nodejs.win.prep_11} args: tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-node-tests.sh 11.x" metadata: diff --git a/taskcluster/test-nodejs_4x-win-amd64-opt.yml b/taskcluster/test-nodejs_4x-win-amd64-opt.yml index 9101502e..04721db6 100644 --- a/taskcluster/test-nodejs_4x-win-amd64-opt.yml +++ b/taskcluster/test-nodejs_4x-win-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.win.sox} && ${nodejs.win.prep_4} + ${system.sox_win} && ${nodejs.win.prep_4} args: tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-node-tests.sh 4.x" metadata: diff --git a/taskcluster/test-nodejs_6x-win-amd64-opt.yml b/taskcluster/test-nodejs_6x-win-amd64-opt.yml index e58a1955..5d9bc68d 100644 --- a/taskcluster/test-nodejs_6x-win-amd64-opt.yml +++ b/taskcluster/test-nodejs_6x-win-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.win.sox} && ${nodejs.win.prep_6} + ${system.sox_win} && ${nodejs.win.prep_6} args: tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-node-tests.sh 6.x" metadata: diff --git a/taskcluster/test-nodejs_7x-win-amd64-opt.yml b/taskcluster/test-nodejs_7x-win-amd64-opt.yml index 4182691e..bdbee9b0 100644 --- a/taskcluster/test-nodejs_7x-win-amd64-opt.yml +++ b/taskcluster/test-nodejs_7x-win-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.win.sox} && ${nodejs.win.prep_7} + ${system.sox_win} && ${nodejs.win.prep_7} args: tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-node-tests.sh 7.x" metadata: diff --git a/taskcluster/test-nodejs_8x-win-amd64-opt.yml b/taskcluster/test-nodejs_8x-win-amd64-opt.yml index e3c8f720..0d5bf6b2 100644 --- a/taskcluster/test-nodejs_8x-win-amd64-opt.yml +++ b/taskcluster/test-nodejs_8x-win-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.win.sox} && ${nodejs.win.prep_8} + ${system.sox_win} && ${nodejs.win.prep_8} args: tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-node-tests.sh 8.x" metadata: diff --git a/taskcluster/test-nodejs_9x-win-amd64-opt.yml b/taskcluster/test-nodejs_9x-win-amd64-opt.yml index fa6d0f14..8da72bb7 100644 --- a/taskcluster/test-nodejs_9x-win-amd64-opt.yml +++ b/taskcluster/test-nodejs_9x-win-amd64-opt.yml @@ -5,7 +5,7 @@ build: - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.win.sox} && ${nodejs.win.prep_9} + ${system.sox_win} && ${nodejs.win.prep_9} args: tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-node-tests.sh 9.x" metadata: diff --git a/taskcluster/test-python_35-win-amd64-opt.yml b/taskcluster/test-python_35-win-amd64-opt.yml new file mode 100644 index 00000000..647b18b4 --- /dev/null +++ b/taskcluster/test-python_35-win-amd64-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-cpu-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${system.sox_win} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-python-tests.sh 3.5.4:m" + metadata: + name: "DeepSpeech Windows AMD64 CPU Python v3.5 tests" + description: "Testing DeepSpeech for Windows/AMD64 on Python v3.5, CPU only, optimized version" diff --git a/taskcluster/test-python_36-win-amd64-opt.yml b/taskcluster/test-python_36-win-amd64-opt.yml new file mode 100644 index 00000000..3b58d3cd --- /dev/null +++ b/taskcluster/test-python_36-win-amd64-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-cpu-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${system.sox_win} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-python-tests.sh 3.6.7:m" + metadata: + name: "DeepSpeech Windows AMD64 CPU Python v3.6 tests" + description: "Testing DeepSpeech for Windows/AMD64 on Python v3.6, CPU only, optimized version" diff --git a/taskcluster/test-python_37-win-amd64-opt.yml b/taskcluster/test-python_37-win-amd64-opt.yml new file mode 100644 index 00000000..22378a74 --- /dev/null +++ b/taskcluster/test-python_37-win-amd64-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-cpu-opt" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${system.sox_win} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/tc-python-tests.sh 3.7.1:m" + metadata: + name: "DeepSpeech Windows AMD64 CPU Python v3.7 tests" + description: "Testing DeepSpeech for Windows/AMD64 on Python v3.7, CPU only, optimized version" diff --git a/taskcluster/win-build.sh b/taskcluster/win-build.sh index c1232d46..dce577b0 100755 --- a/taskcluster/win-build.sh +++ b/taskcluster/win-build.sh @@ -35,6 +35,10 @@ export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin:'/c/Program Files/nodejs/' do_deepspeech_binary_build +# Those are the versions available on NuGet.org +export SUPPORTED_PYTHON_VERSIONS="3.5.4 3.6.7 3.7.1" +do_deepspeech_python_build "${cuda}" + do_deepspeech_nodejs_build "${cuda}" do_deepspeech_netframework_build diff --git a/taskcluster/win-package.sh b/taskcluster/win-package.sh index a1f63b3b..16983755 100755 --- a/taskcluster/win-package.sh +++ b/taskcluster/win-package.sh @@ -16,6 +16,10 @@ cp ${DS_ROOT_TASK}/DeepSpeech/ds/examples/net_framework/CSharpExamples/*.nupkg $ cp ${DS_ROOT_TASK}/DeepSpeech/ds/examples/net_framework/CSharpExamples/DeepSpeechConsole/bin/x64/Release/DeepSpeechConsole.exe ${TASKCLUSTER_ARTIFACTS}/ +if [ -d ${DS_ROOT_TASK}/DeepSpeech/ds/wheels ]; then + cp ${DS_ROOT_TASK}/DeepSpeech/ds/wheels/* ${TASKCLUSTER_ARTIFACTS}/ +fi; + if [ -f ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/wrapper.tar.gz ]; then cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/wrapper.tar.gz ${TASKCLUSTER_ARTIFACTS}/ cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/deepspeech-*.tgz ${TASKCLUSTER_ARTIFACTS}/ diff --git a/tc-python-tests-prod.sh b/tc-python-tests-prod.sh index 46fe6069..6803082e 100644 --- a/tc-python-tests-prod.sh +++ b/tc-python-tests-prod.sh @@ -4,24 +4,7 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -pyver_full=$1 - -if [ -z "${pyver_full}" ]; then - echo "No python version given, aborting." - exit 1 -fi; - -pyver=$(echo "${pyver_full}" | cut -d':' -f1) - -# 2.7.x => 27 -pyver_pkg=$(echo "${pyver}" | cut -d'.' -f1,2 | tr -d '.') - -py_unicode_type=$(echo "${pyver_full}" | cut -d':' -f2) -if [ "${py_unicode_type}" = "m" ]; then - pyconf="ucs2" -elif [ "${py_unicode_type}" = "mu" ]; then - pyconf="ucs4" -fi; +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" unset PYTHON_BIN_PATH unset PYTHONPATH @@ -43,18 +26,17 @@ install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" maybe_ssl102_py37 ${pyver} +maybe_numpy_min_version_winamd64 ${pyver} + PYENV_NAME=deepspeech-test -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv install ${pyver} -pyenv virtualenv ${pyver} ${PYENV_NAME} -source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} -platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') -whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')" -deepspeech_pkg="deepspeech-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" +setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}" +virtualenv_activate "${pyver}" "${PYENV_NAME}" -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${DEEPSPEECH_ARTIFACTS_ROOT}/${deepspeech_pkg} | cat +deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type}) +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat run_prod_inference_tests -deactivate -pyenv uninstall --force ${PYENV_NAME} +virtualenv_deactivate "${pyver}" "${PYENV_NAME}" diff --git a/tc-python-tests.sh b/tc-python-tests.sh index 9227a242..a90bb3c0 100644 --- a/tc-python-tests.sh +++ b/tc-python-tests.sh @@ -4,24 +4,7 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -pyver_full=$1 - -if [ -z "${pyver_full}" ]; then - echo "No python version given, aborting." - exit 1 -fi; - -pyver=$(echo "${pyver_full}" | cut -d':' -f1) - -# 2.7.x => 27 -pyver_pkg=$(echo "${pyver}" | cut -d'.' -f1,2 | tr -d '.') - -py_unicode_type=$(echo "${pyver_full}" | cut -d':' -f2) -if [ "${py_unicode_type}" = "m" ]; then - pyconf="ucs2" -elif [ "${py_unicode_type}" = "mu" ]; then - pyconf="ucs4" -fi; +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" unset PYTHON_BIN_PATH unset PYTHONPATH @@ -37,19 +20,20 @@ install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" maybe_ssl102_py37 ${pyver} +maybe_numpy_min_version_winamd64 ${pyver} + PYENV_NAME=deepspeech-test -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv install ${pyver} -pyenv virtualenv ${pyver} ${PYENV_NAME} -source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} -platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') -whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')" -deepspeech_pkg="deepspeech-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" +setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}" +virtualenv_activate "${pyver}" "${PYENV_NAME}" -deepspeech_pkg_url=${DEEPSPEECH_ARTIFACTS_ROOT}/${deepspeech_pkg} +deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type}) LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat +which deepspeech +deepspeech --version + run_all_inference_tests -deactivate -pyenv uninstall --force ${PYENV_NAME} +virtualenv_deactivate "${pyver}" "${PYENV_NAME}" diff --git a/tc-single-shot-inference.sh b/tc-single-shot-inference.sh index 7f3f9fe5..43790dff 100755 --- a/tc-single-shot-inference.sh +++ b/tc-single-shot-inference.sh @@ -4,27 +4,11 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -pyver_full=$1 +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" + ds=$2 frozen=$2 -if [ -z "${pyver_full}" ]; then - echo "No python version given, aborting." - exit 1 -fi; - -pyver=$(echo "${pyver_full}" | cut -d':' -f1) - -# 2.7.x => 27 -pyver_pkg=$(echo "${pyver}" | cut -d'.' -f1,2 | tr -d '.') - -py_unicode_type=$(echo "${pyver_full}" | cut -d':' -f2) -if [ "${py_unicode_type}" = "m" ]; then - pyconf="ucs2" -elif [ "${py_unicode_type}" = "mu" ]; then - pyconf="ucs4" -fi; - unset PYTHON_BIN_PATH unset PYTHONPATH export PYENV_ROOT="${HOME}/ds-train/.pyenv" @@ -39,23 +23,18 @@ install_pyenv "${PYENV_ROOT}" install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" PYENV_NAME=deepspeech-train -PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver} -pyenv virtualenv ${pyver} ${PYENV_NAME} -source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate +PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv_install ${pyver} + +setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}" +virtualenv_activate "${pyver}" "${PYENV_NAME}" pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt | cat -platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') -whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')" -decoder_pkg="ds_ctcdecoder-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" - -decoder_pkg_url=${DECODER_ARTIFACTS_ROOT}/${decoder_pkg} - +decoder_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type} "ds_ctcdecoder" "${DECODER_ARTIFACTS_ROOT}") LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${decoder_pkg_url} | cat pushd ${HOME}/DeepSpeech/ds/ time ./bin/run-tc-ldc93s1_singleshotinference.sh popd -deactivate -pyenv uninstall --force ${PYENV_NAME} +virtualenv_deactivate "${pyver}" "${PYENV_NAME}" diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index c114fb6f..9f393001 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -508,6 +508,11 @@ install_pyenv() exit 1; fi; + if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + mkdir -p "${PYENV_ROOT}/versions/" + return; + fi + if [ ! -e "${PYENV_ROOT}/bin/pyenv" ]; then git clone --quiet https://github.com/pyenv/pyenv.git ${PYENV_ROOT} pushd ${PYENV_ROOT} @@ -520,13 +525,18 @@ install_pyenv() install_pyenv_virtualenv() { - PYENV_VENV=$1 + local PYENV_VENV=$1 if [ -z "${PYENV_VENV}" ]; then echo "No PYENV_VENV set"; exit 1; fi; + if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + echo "No pyenv virtualenv support ; will install virtualenv locally from pip" + return + fi; + if [ ! -e "${PYENV_VENV}/bin/pyenv-virtualenv" ]; then git clone --quiet https://github.com/pyenv/pyenv-virtualenv.git ${PYENV_VENV} pushd ${PYENV_VENV} @@ -537,6 +547,86 @@ install_pyenv_virtualenv() eval "$(pyenv virtualenv-init -)" } +setup_pyenv_virtualenv() +{ + local version=$1 + local name=$2 + + if [ -z "${PYENV_ROOT}" ]; then + echo "No PYENV_ROOT set"; + exit 1; + fi; + + if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + echo "installing virtualenv" + PATH=${PYENV_ROOT}/versions/python.${version}/tools:${PYENV_ROOT}/versions/python.${version}/tools/Scripts:$PATH pip install virtualenv + + echo "should setup virtualenv ${name} for ${version}" + mkdir ${PYENV_ROOT}/versions/python.${version}/envs + PATH=${PYENV_ROOT}/versions/python.${version}/tools:${PYENV_ROOT}/versions/python.${version}/tools/Scripts:$PATH virtualenv ${PYENV_ROOT}/versions/python.${version}/envs/${name} + else + pyenv virtualenv ${version} ${name} + fi +} + +virtualenv_activate() +{ + local version=$1 + local name=$2 + + if [ -z "${PYENV_ROOT}" ]; then + echo "No PYENV_ROOT set"; + exit 1; + fi; + + if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + source ${PYENV_ROOT}/versions/python.${version}/envs/${name}/Scripts/activate + else + source ${PYENV_ROOT}/versions/${version}/envs/${name}/bin/activate + fi +} + +virtualenv_deactivate() +{ + local version=$1 + local name=$2 + + if [ -z "${PYENV_ROOT}" ]; then + echo "No PYENV_ROOT set"; + exit 1; + fi; + + deactivate + + if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + rm -fr ${PYENV_ROOT}/versions/python.${version}/ + else + pyenv uninstall --force ${name} + pyenv uninstall --force ${version} + fi +} + +pyenv_install() +{ + local version=$1 + + if [ -z "${PYENV_ROOT}" ]; then + echo "No PYENV_ROOT set"; + exit 1; + fi; + + if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + PATH=$(cygpath ${ChocolateyInstall})/bin:$PATH nuget install python -Version ${version} -OutputDirectory ${PYENV_ROOT}/versions/ + PATH=${PYENV_ROOT}/versions/python.${version}/tools/:$PATH python -m pip uninstall pip -y + PATH=${PYENV_ROOT}/versions/python.${version}/tools/:$PATH python -m ensurepip + pushd ${PYENV_ROOT}/versions/python.${version}/tools/Scripts/ + ln -s pip3.exe pip.exe + popd + else + pyenv install ${version} + fi +} + maybe_install_xldd() { # -s required to avoid the noisy output like "Entering / Leaving directories" @@ -806,6 +896,82 @@ maybe_ssl102_py37() esac } +maybe_numpy_min_version_winamd64() +{ + local pyver=$1 + + if [ "${OS}" != "${TC_MSYS_VERSION}" ]; then + return; + fi + + # We set >= and < to make sure we have no numpy incompatibilities + # otherwise, `from deepspeech.impl` throws with "illegal instruction" + case "${pyver}" in + 3.5*) + export NUMPY_BUILD_VERSION="==1.11.0" + export NUMPY_DEP_VERSION=">=1.11.0,<1.12.0" + ;; + 3.6*) + export NUMPY_BUILD_VERSION="==1.12.0" + export NUMPY_DEP_VERSION=">=1.12.0,<1.14.5" + ;; + 3.7*) + export NUMPY_BUILD_VERSION="==1.14.5" + export NUMPY_DEP_VERSION=">=1.14.5,<1.16.0" + ;; + esac +} + +get_python_pkg_url() +{ + local pyver_pkg=$1 + local py_unicode_type=$2 + + local pkgname=$3 + if [ -z "${pkgname}" ]; then + pkgname="deepspeech" + fi + + local root=$4 + if [ -z "${root}" ]; then + root="${DEEPSPEECH_ARTIFACTS_ROOT}" + fi + + local platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; plat = "win" if plat == "windows" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine().lower()));') + local whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')" + local deepspeech_pkg="${pkgname}-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" + + echo "${root}/${deepspeech_pkg}" +} + +extract_python_versions() +{ + # call extract_python_versions ${pyver_full} pyver pyver_pkg py_unicode_type pyconf + local _pyver_full=$1 + + if [ -z "${_pyver_full}" ]; then + echo "No python version given, aborting." + exit 1 + fi; + + local _pyver=$(echo "${_pyver_full}" | cut -d':' -f1) + + # 2.7.x => 27 + local _pyver_pkg=$(echo "${_pyver}" | cut -d'.' -f1,2 | tr -d '.') + + local _py_unicode_type=$(echo "${_pyver_full}" | cut -d':' -f2) + if [ "${_py_unicode_type}" = "m" ]; then + local _pyconf="ucs2" + elif [ "${_py_unicode_type}" = "mu" ]; then + local _pyconf="ucs4" + fi; + + eval "${2}=${_pyver}" + eval "${3}=${_pyver_pkg}" + eval "${4}=${_py_unicode_type}" + eval "${5}=${_pyconf}" +} + do_deepspeech_python_build() { cd ${DS_DSDIR} @@ -836,10 +1002,12 @@ do_deepspeech_python_build() maybe_ssl102_py37 ${pyver} - LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" pyenv install ${pyver} + maybe_numpy_min_version_winamd64 ${pyver} - pyenv virtualenv ${pyver} deepspeech - source ${PYENV_ROOT}/versions/${pyver}/envs/deepspeech/bin/activate + LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" pyenv_install ${pyver} + + setup_pyenv_virtualenv "${pyver}" "deepspeech" + virtualenv_activate "${pyver}" "deepspeech" # Set LD path because python ssl might require it LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ @@ -860,9 +1028,7 @@ do_deepspeech_python_build() unset NUMPY_BUILD_VERSION unset NUMPY_DEP_VERSION - deactivate - pyenv uninstall --force deepspeech - pyenv uninstall --force ${pyver} + virtualenv_deactivate "${pyver}" "deepspeech" done; }