From 8029f3d7dd081aa52944fbc165005b3f1aa03014 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 20 Feb 2020 13:20:26 +0100 Subject: [PATCH] Produce pyenv ready-to-use --- requirements.txt | 2 +- taskcluster/.shared.yml | 15 ++- taskcluster/android-arm64-cpu-opt.yml | 1 + taskcluster/android-armv7-cpu-opt.yml | 1 + taskcluster/android-java-opt.yml | 14 +- taskcluster/android-x86_64-cpu-opt.yml | 1 + taskcluster/darwin-amd64-cpu-opt.yml | 1 + taskcluster/darwin-amd64-ctc-opt.yml | 1 + taskcluster/darwin-amd64-tflite-opt.yml | 1 + taskcluster/darwin-opt-base.tyml | 64 +++------ taskcluster/linux-amd64-cpu-opt.yml | 4 +- taskcluster/linux-amd64-ctc-opt.yml | 4 +- taskcluster/linux-amd64-gpu-opt.yml | 4 +- taskcluster/linux-amd64-tflite-opt.yml | 4 +- taskcluster/linux-arm64-cpu-opt.yml | 6 +- taskcluster/linux-opt-base.tyml | 7 +- taskcluster/linux-rpi3-cpu-opt.yml | 6 +- taskcluster/node-package-cpu.yml | 6 +- taskcluster/node-package-gpu.yml | 6 +- taskcluster/node-package-opt-base.tyml | 66 +++++++++ taskcluster/node-package-tflite.yml | 6 +- taskcluster/pyenv-build.sh | 37 +++++ taskcluster/pyenv-darwin-amd64.yml | 8 ++ taskcluster/pyenv-darwin-opt-base.tyml | 55 ++++++++ taskcluster/pyenv-linux-amd64.yml | 11 ++ taskcluster/pyenv-linux-opt-base.tyml | 52 ++++++++ taskcluster/pyenv-package.sh | 18 +++ taskcluster/pyenv-win-amd64.yml | 8 ++ taskcluster/pyenv-win-opt-base.tyml | 58 ++++++++ taskcluster/tc-all-vars.sh | 7 + taskcluster/tc-build-utils.sh | 60 +-------- taskcluster/tc-evaluate_tflite.sh | 30 +---- taskcluster/tc-py-utils.sh | 126 ++++++++++++------ taskcluster/tc-python-tests-prod.sh | 30 +---- taskcluster/tc-python-tests.sh | 30 +---- taskcluster/tc-python_tflite-tests-prod.sh | 30 +---- taskcluster/tc-python_tflite-tests.sh | 30 +---- taskcluster/tc-single-shot-inference.sh | 19 +-- taskcluster/tc-train-tests.sh | 43 +----- taskcluster/tc-transfer-tests.sh | 42 +----- taskcluster/test-darwin-opt-base.tyml | 15 ++- ...-evaluate_tflite-linux-amd64-py36m-opt.yml | 5 +- taskcluster/test-linux-opt-base.tyml | 2 +- .../test-python_35_16k-linux-amd64-opt.yml | 3 - ...on_35_16k-linux-amd64-prod_pbmodel-opt.yml | 3 - .../test-python_35_8k-linux-amd64-opt.yml | 3 - ...hon_35_8k-linux-amd64-prod_pbmodel-opt.yml | 3 - ...hon_35_tflite_16k-linux-amd64-prod-opt.yml | 3 - ...thon_35_tflite_8k-linux-amd64-prod-opt.yml | 3 - .../test-python_36_16k-linux-amd64-opt.yml | 3 - ...on_36_16k-linux-amd64-prod_pbmodel-opt.yml | 3 - .../test-python_36_8k-linux-amd64-opt.yml | 3 - ...hon_36_8k-linux-amd64-prod_pbmodel-opt.yml | 3 - ...hon_36_tflite_16k-linux-amd64-prod-opt.yml | 3 - ...thon_36_tflite_8k-linux-amd64-prod-opt.yml | 3 - .../test-python_37_16k-linux-amd64-opt.yml | 3 - ...on_37_16k-linux-amd64-prod_pbmodel-opt.yml | 3 - .../test-python_37_8k-linux-amd64-opt.yml | 3 - ...hon_37_8k-linux-amd64-prod_pbmodel-opt.yml | 3 - ...hon_37_tflite_16k-linux-amd64-prod-opt.yml | 3 - ...thon_37_tflite_8k-linux-amd64-prod-opt.yml | 3 - .../test-python_38_16k-linux-amd64-opt.yml | 3 - ...on_38_16k-linux-amd64-prod_pbmodel-opt.yml | 3 - .../test-python_38_8k-linux-amd64-opt.yml | 3 - ...hon_38_8k-linux-amd64-prod_pbmodel-opt.yml | 3 - ...hon_38_tflite_16k-linux-amd64-prod-opt.yml | 3 - ...thon_38_tflite_8k-linux-amd64-prod-opt.yml | 3 - ...gleshotinference-linux-amd64-py36m-opt.yml | 5 +- ...est-training_16k-linux-amd64-py35m-opt.yml | 5 +- ...est-training_16k-linux-amd64-py36m-opt.yml | 5 +- ...test-training_8k-linux-amd64-py36m-opt.yml | 5 +- .../test-transfer-linux-amd64-py36m-opt.yml | 5 +- taskcluster/test-win-opt-base.tyml | 17 ++- taskcluster/win-amd64-cpu-opt.yml | 1 + taskcluster/win-amd64-gpu-opt.yml | 1 + taskcluster/win-amd64-tflite-opt.yml | 1 + taskcluster/win-opt-base.tyml | 39 +++--- 77 files changed, 563 insertions(+), 529 deletions(-) create mode 100644 taskcluster/node-package-opt-base.tyml create mode 100755 taskcluster/pyenv-build.sh create mode 100644 taskcluster/pyenv-darwin-amd64.yml create mode 100644 taskcluster/pyenv-darwin-opt-base.tyml create mode 100644 taskcluster/pyenv-linux-amd64.yml create mode 100644 taskcluster/pyenv-linux-opt-base.tyml create mode 100755 taskcluster/pyenv-package.sh create mode 100644 taskcluster/pyenv-win-amd64.yml create mode 100644 taskcluster/pyenv-win-opt-base.tyml diff --git a/requirements.txt b/requirements.txt index 69805d58..d8a71c3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ semver setuptools # Requirements for transcribe.py -webrtcvad +webrtcvad[transcribe] # Requirements for importers sox diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 63ebb7b7..ce5445bf 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -2,9 +2,12 @@ python: packages_trusty: apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libpng-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev' packages_buster: - apt: 'make build-essential libssl-dev libffi-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev' + apt: 'python3-virtualenv python3-setuptools python3-pip python3-wheel python3-pkg-resources' packages_docs_bionic: apt: 'python3 python3-pip zip doxygen' +tensorflow: + packages_trusty: + apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python zlib1g-dev' electronjs: packages_xenial: apt: 'libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xvfb' @@ -52,6 +55,16 @@ system: homebrew_tests: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz' namespace: 'project.deepspeech.homebrew_tests.4' + pyenv: + linux: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.5/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.linux.5' + osx: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.osx.5/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.osx.5' + win: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.5/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.win.5' swig: repo: "https://github.com/lissyx/swig" sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118" diff --git a/taskcluster/android-arm64-cpu-opt.yml b/taskcluster/android-arm64-cpu-opt.yml index cabba0c7..fcd1b2be 100644 --- a/taskcluster/android-arm64-cpu-opt.yml +++ b/taskcluster/android-arm64-cpu-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64" diff --git a/taskcluster/android-armv7-cpu-opt.yml b/taskcluster/android-armv7-cpu-opt.yml index 8a7f09bb..2578758b 100644 --- a/taskcluster/android-armv7-cpu-opt.yml +++ b/taskcluster/android-armv7-cpu-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7" diff --git a/taskcluster/android-java-opt.yml b/taskcluster/android-java-opt.yml index 5fbad0fa..69f44dc7 100644 --- a/taskcluster/android-java-opt.yml +++ b/taskcluster/android-java-opt.yml @@ -1,16 +1,20 @@ build: template_file: linux-opt-base.tyml + dependencies: + - "android-arm64-cpu-opt" + - "android-armv7-cpu-opt" + - "android-x86_64-cpu-opt" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-apk" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-apk" - "index.project.deepspeech.deepspeech.native_client.android-apk.${event.head.sha}" - dependencies: - - "android-arm64-cpu-opt" - - "android-armv7-cpu-opt" - - "android-x86_64-cpu-opt" system_setup: > - apt-get -qq -y install curl + apt-get -qq -y install curl software-properties-common && + add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && + DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && + java -version && update-ca-certificates -f tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.android-armv7/artifacts/public/home.tar.xz" scripts: build: "taskcluster/android-apk-build.sh" diff --git a/taskcluster/android-x86_64-cpu-opt.yml b/taskcluster/android-x86_64-cpu-opt.yml index 0e3114a5..447822ab 100644 --- a/taskcluster/android-x86_64-cpu-opt.yml +++ b/taskcluster/android-x86_64-cpu-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64" diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index 211445c9..85894360 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -4,6 +4,7 @@ build: - "swig-darwin-amd64" - "node-gyp-cache" - "homebrew_builds-darwin-amd64" + - "pyenv-darwin-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx" diff --git a/taskcluster/darwin-amd64-ctc-opt.yml b/taskcluster/darwin-amd64-ctc-opt.yml index a1ce32c6..0e3e51e6 100644 --- a/taskcluster/darwin-amd64-ctc-opt.yml +++ b/taskcluster/darwin-amd64-ctc-opt.yml @@ -4,6 +4,7 @@ build: - "swig-darwin-amd64" - "node-gyp-cache" - "homebrew_builds-darwin-amd64" + - "pyenv-darwin-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc" diff --git a/taskcluster/darwin-amd64-tflite-opt.yml b/taskcluster/darwin-amd64-tflite-opt.yml index c901d88a..4ab6c7c7 100644 --- a/taskcluster/darwin-amd64-tflite-opt.yml +++ b/taskcluster/darwin-amd64-tflite-opt.yml @@ -4,6 +4,7 @@ build: - "swig-darwin-amd64" - "node-gyp-cache" - "homebrew_builds-darwin-amd64" + - "pyenv-darwin-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite" diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index e1519bc7..bb37a6c4 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -26,11 +26,6 @@ routes: then: { $eval: build.routes } -scopes: - $if: '(event.event != "push") && (event.event != "tag")' - then: - - generic-worker:cache:deepspeech-macos-pyenv - payload: maxRunTime: { $eval: to_int(build.maxRunTime) } @@ -53,8 +48,8 @@ payload: - "-cxe" - > export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && - export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && cd ../tc-workdir/ && - (mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) && + export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && + (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && export TASKCLUSTER_TASK_DIR="$(pwd)" && export LC_ALL=C && export MACOSX_DEPLOYMENT_TARGET=10.10 && @@ -62,7 +57,6 @@ payload: export HOMEBREW_NO_AUTO_UPDATE=1 && env && trap "export TASKCLUSTER_TASK_EXIT_CODE=$? && - (mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ || true) && cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ && exit $TASKCLUSTER_TASK_EXIT_CODE" 0 && (pixz -d < $TASKCLUSTER_ORIG_TASKDIR/home.tar.xz | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) && @@ -70,6 +64,8 @@ payload: cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client && cd $TASKCLUSTER_TASK_DIR && + (mkdir pyenv-root/ && gtar -C pyenv-root/ -xf $TASKCLUSTER_ORIG_TASKDIR/pyenv.tar.gz) && + (mkdir homebrew-builds/ && gtar -C homebrew-builds/ -xf $TASKCLUSTER_ORIG_TASKDIR/homebrew-builds.tar.gz) && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} @@ -82,41 +78,23 @@ payload: else: { $fromNow: '7 days' } mounts: - $if: '(event.event != "push") && (event.event != "tag")' - then: - - cacheName: deepspeech-macos-pyenv - directory: pyenv.cache/ - - directory: ${system.homedir.osx}/Library/Caches/node-gyp/ - format: tar.gz - content: - url: ${system.node_gyp_cache.url} - - directory: ds-swig - format: tar.gz - content: - url: ${system.swig.cache.darwin_amd64} - - file: home.tar.xz - content: - url: ${build.tensorflow} - - directory: ../tc-workdir/homebrew-builds - format: tar.gz - content: - url: ${system.homebrew_builds.url} - else: - - directory: ${system.homedir.osx}/Library/Caches/node-gyp/ - format: tar.gz - content: - url: ${system.node_gyp_cache.url} - - directory: ds-swig - format: tar.gz - content: - url: ${system.swig.cache.darwin_amd64} - - file: home.tar.xz - content: - url: ${build.tensorflow} - - directory: ../tc-workdir/homebrew-builds - format: tar.gz - content: - url: ${system.homebrew_builds.url} + - file: pyenv.tar.gz + content: + url: ${system.pyenv.osx.url} + - file: homebrew-builds.tar.gz + content: + url: ${system.homebrew_builds.url} + - directory: ${system.homedir.osx}/Library/Caches/node-gyp/ + format: tar.gz + content: + url: ${system.node_gyp_cache.url} + - directory: ds-swig + format: tar.gz + content: + url: ${system.swig.cache.darwin_amd64} + - file: home.tar.xz + content: + url: ${build.tensorflow} metadata: name: ${build.metadata.name} diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index ede232cf..e2af482b 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu" @@ -10,8 +11,7 @@ build: system_setup: > ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" scripts: build: "taskcluster/host-build.sh" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml index 2348e0bc..00af6149 100644 --- a/taskcluster/linux-amd64-ctc-opt.yml +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-ctc" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-ctc" @@ -10,8 +11,7 @@ build: system_setup: > ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" scripts: build: 'taskcluster/decoder-build.sh' diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index a4541656..532b8bb7 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu" @@ -10,8 +11,7 @@ build: system_setup: > ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.gpu/artifacts/public/home.tar.xz" maxRunTime: 14400 scripts: diff --git a/taskcluster/linux-amd64-tflite-opt.yml b/taskcluster/linux-amd64-tflite-opt.yml index ec2dfa44..3d4847f9 100644 --- a/taskcluster/linux-amd64-tflite-opt.yml +++ b/taskcluster/linux-amd64-tflite-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite" @@ -10,8 +11,7 @@ build: system_setup: > ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" scripts: build: "taskcluster/host-build.sh tflite" diff --git a/taskcluster/linux-arm64-cpu-opt.yml b/taskcluster/linux-arm64-cpu-opt.yml index 3faa61ac..1865784e 100644 --- a/taskcluster/linux-arm64-cpu-opt.yml +++ b/taskcluster/linux-arm64-cpu-opt.yml @@ -3,11 +3,11 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64" - "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm64/artifacts/public/home.tar.xz" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > @@ -15,11 +15,11 @@ build: wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb && ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml system_config: > multistrap -d /tmp/multistrap-armbian64-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_armbian64_buster.conf + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm64/artifacts/public/home.tar.xz" scripts: build: "taskcluster/arm64-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index edebf5fc..dd50397a 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -46,11 +46,10 @@ then: apt-get -qq -y remove --purge ubuntu-advantage-tools && ${aptEc2Mirrors} && adduser --system --home ${system.homedir.linux} ${system.username} && - apt-get -qq update && apt-get -qq -y install git pixz wget pkg-config libsox-dev && ${extraSystemSetup} && + apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_trusty.apt} pixz pkg-config realpath unzip wget zip && ${extraSystemSetup} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && - ${system.homedir.linux}/DeepSpeech/tf/tc-apt.sh && ${extraSystemConfig} && + echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 9f69f26e..0cabb870 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -3,11 +3,11 @@ build: dependencies: - "swig-linux-amd64" - "node-gyp-cache" + - "pyenv-linux-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm" - "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm/artifacts/public/home.tar.xz" ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 system_setup: > @@ -15,11 +15,11 @@ build: wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb && ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml system_config: > multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm/artifacts/public/home.tar.xz" scripts: build: "taskcluster/rpi3-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/node-package-cpu.yml b/taskcluster/node-package-cpu.yml index d8ea192c..72870cd5 100644 --- a/taskcluster/node-package-cpu.yml +++ b/taskcluster/node-package-cpu.yml @@ -1,5 +1,5 @@ build: - template_file: linux-opt-base.tyml + template_file: node-package-opt-base.tyml dependencies: - "darwin-amd64-cpu-opt" - "linux-amd64-cpu-opt" @@ -9,9 +9,7 @@ build: system_setup: > ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml scripts: build: "taskcluster/node-build.sh" package: "taskcluster/node-package.sh" diff --git a/taskcluster/node-package-gpu.yml b/taskcluster/node-package-gpu.yml index b78e72d1..db48ae32 100644 --- a/taskcluster/node-package-gpu.yml +++ b/taskcluster/node-package-gpu.yml @@ -1,14 +1,12 @@ build: - template_file: linux-opt-base.tyml + template_file: node-package-opt-base.tyml dependencies: - "linux-amd64-gpu-opt" - "win-amd64-gpu-opt" system_setup: > ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml scripts: build: "taskcluster/node-build.sh --cuda" package: "taskcluster/node-package.sh" diff --git a/taskcluster/node-package-opt-base.tyml b/taskcluster/node-package-opt-base.tyml new file mode 100644 index 00000000..b17daaac --- /dev/null +++ b/taskcluster/node-package-opt-base.tyml @@ -0,0 +1,66 @@ +$if: 'event.event in build.allowed' +then: + taskId: ${taskcluster.taskId} + provisionerId: ${taskcluster.docker.provisionerId} + workerType: ${taskcluster.docker.workerType} + taskGroupId: ${taskcluster.taskGroupId} + schedulerId: ${taskcluster.schedulerId} + dependencies: + $map: { $eval: build.dependencies } + each(b): + $eval: as_slugid(b) + created: { $fromNow: '0 sec' } + deadline: { $fromNow: '1 day' } + expires: + $if: '(event.event == "push") || (event.event == "tag")' + then: { $fromNow: '6 months' } + else: { $fromNow: '7 days' } + + extra: + nc_asset_name: { $eval: build.nc_asset_name } + github: + $if: '(event.event == "push") || (event.event == "tag")' + then: { $eval: taskcluster.github_events.merge } + else: { $eval: taskcluster.github_events.pull_request } + + routes: + $if: '(event.event == "push") || (event.event == "tag")' + then: + { $eval: build.routes } + + payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + image: "ubuntu:14.04" + + command: + - "/bin/bash" + - "--login" + - "-cxe" + - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + extraSystemConfig: { $eval: strip(str(build.system_config)) } + in: > + apt-get -qq -y remove --purge ubuntu-advantage-tools && + ${aptEc2Mirrors} && + adduser --system --home ${system.homedir.linux} ${system.username} && + apt-get -qq update && apt-get -qq -y install realpath git wget curl make && ${extraSystemSetup} && + cd ${system.homedir.linux}/ && + echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ~/DeepSpeech/tf/ && touch ~/DeepSpeech/tf/tc-vars.sh && chmod +x ~/DeepSpeech/tf/tc-vars.sh && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && + sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} + + artifacts: + "public": + type: "directory" + path: "/tmp/artifacts/" + expires: + $if: '(event.event == "push") || (event.event == "tag")' + then: { $fromNow: '6 months' } + else: { $fromNow: '7 days' } + + metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/node-package-tflite.yml b/taskcluster/node-package-tflite.yml index e9fa2259..d49404d0 100644 --- a/taskcluster/node-package-tflite.yml +++ b/taskcluster/node-package-tflite.yml @@ -1,5 +1,5 @@ build: - template_file: linux-opt-base.tyml + template_file: node-package-opt-base.tyml dependencies: - "darwin-amd64-tflite-opt" - "linux-amd64-tflite-opt" @@ -7,9 +7,7 @@ build: system_setup: > ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} - && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && - apt-get -qq -y install ${python.packages_trusty.apt} - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" + && apt-get -qq update && apt-get -qq -y install nodejs python-yaml scripts: build: "taskcluster/node-build.sh --tflite" package: "taskcluster/node-package.sh" diff --git a/taskcluster/pyenv-build.sh b/taskcluster/pyenv-build.sh new file mode 100755 index 00000000..b3faf8fb --- /dev/null +++ b/taskcluster/pyenv-build.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +unset PYTHON_BIN_PATH +unset PYTHONPATH + +export PATH="${PYENV_ROOT}/bin:$PATH" + +install_pyenv "${PYENV_ROOT}" +install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" + +for pyver_conf in ${SUPPORTED_PYTHON_VERSIONS}; do + pyver=$(echo "${pyver_conf}" | cut -d':' -f1) + pyconf=$(echo "${pyver_conf}" | cut -d':' -f2) + + pyalias="${pyver}_${pyconf}" + + maybe_ssl102_py37 ${pyver} + + LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ + PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \ + pyenv_install ${pyver} ${pyalias} + + setup_pyenv_virtualenv "${pyalias}" "deepspeech" + + virtualenv_activate "${pyalias}" "deepspeech" + + python --version + python3 --version || true # Might fail without any issue on Windows + which pip + which pip3 || true # Might fail without any issue on Windows + + virtualenv_deactivate "${pyalias}" "deepspeech" +done; diff --git a/taskcluster/pyenv-darwin-amd64.yml b/taskcluster/pyenv-darwin-amd64.yml new file mode 100644 index 00000000..47cceec9 --- /dev/null +++ b/taskcluster/pyenv-darwin-amd64.yml @@ -0,0 +1,8 @@ +build: + template_file: pyenv-darwin-opt-base.tyml + scripts: + build: "taskcluster/pyenv-build.sh" + package: "taskcluster/pyenv-package.sh" + metadata: + name: "Builds Pyenv macOS AMD64" + description: "Setup a builds Pyenv for macOS/AMD64" diff --git a/taskcluster/pyenv-darwin-opt-base.tyml b/taskcluster/pyenv-darwin-opt-base.tyml new file mode 100644 index 00000000..063c6613 --- /dev/null +++ b/taskcluster/pyenv-darwin-opt-base.tyml @@ -0,0 +1,55 @@ +taskId: ${taskcluster.taskId} +provisionerId: ${taskcluster.generic.provisionerId} +workerType: ${taskcluster.generic.workerType} +taskGroupId: ${taskcluster.taskGroupId} +schedulerId: ${taskcluster.schedulerId} +dependencies: + $map: { $eval: build.dependencies } + each(b): + $eval: as_slugid(b) +created: { $fromNow: '0 sec' } +deadline: { $fromNow: '1 day' } +expires: { $fromNow: '6 months' } +scopes: + - "index:insert-task:project.deepspeech.*" + +payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + + features: + taskclusterProxy: true + + command: + - - "/bin/bash" + - "--login" + - "-cxe" + - $let: + taskIndexExpire: { $fromNow: '6 months' } + in: > + export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && + export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && + (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && + export TASKCLUSTER_TASK_DIR="$(pwd)" && + export LC_ALL=C && + export MACOSX_DEPLOYMENT_TARGET=10.10 && + export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && + env && + mkdir -p $TASKCLUSTER_ARTIFACTS/ && + swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.osx.url}` && + if [ "$swig_bin" != "200" ]; then + git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && + cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} 127.0.0.1:8080 ${system.pyenv.osx.namespace} + fi; + + artifacts: + - type: "directory" + path: "public/" + expires: { $fromNow: '6 months' } + +metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/pyenv-linux-amd64.yml b/taskcluster/pyenv-linux-amd64.yml new file mode 100644 index 00000000..393a8872 --- /dev/null +++ b/taskcluster/pyenv-linux-amd64.yml @@ -0,0 +1,11 @@ +build: + template_file: pyenv-linux-opt-base.tyml + system_setup: + > + apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_trusty.apt} wget + scripts: + build: "taskcluster/pyenv-build.sh" + package: "taskcluster/pyenv-package.sh" + metadata: + name: "Builds Pyenv Linux AMD64" + description: "Setup a builds Pyenv for Linux/AMD64" diff --git a/taskcluster/pyenv-linux-opt-base.tyml b/taskcluster/pyenv-linux-opt-base.tyml new file mode 100644 index 00000000..15d8f7b0 --- /dev/null +++ b/taskcluster/pyenv-linux-opt-base.tyml @@ -0,0 +1,52 @@ +$if: 'event.event in build.allowed' +then: + taskId: ${taskcluster.taskId} + provisionerId: ${taskcluster.docker.provisionerId} + workerType: ${taskcluster.docker.workerType} + taskGroupId: ${taskcluster.taskGroupId} + schedulerId: ${taskcluster.schedulerId} + created: { $fromNow: '0 sec' } + deadline: { $fromNow: '1 day' } + expires: { $fromNow: '6 months' } + scopes: + - "index:insert-task:project.deepspeech.*" + + payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + image: ${build.docker_image} + + features: + taskclusterProxy: true + + command: + - "/bin/bash" + - "--login" + - "-cxe" + - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + taskIndexExpire: { $fromNow: '6 months' } + in: > + (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && + apt-get -qq update && apt-get -qq -y install curl git; + swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.linux.url}` && + if [ "$swig_bin" != "200" ]; then + ${extraSystemSetup} && + adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && + mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && + echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.linux.namespace} + fi; + + artifacts: + "public": + type: "directory" + path: "/tmp/artifacts/" + expires: { $fromNow: '6 months' } + + metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/pyenv-package.sh b/taskcluster/pyenv-package.sh new file mode 100755 index 00000000..bc2c0639 --- /dev/null +++ b/taskcluster/pyenv-package.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -xe + +TC_EXPIRE=$1 +TC_INSTANCE=$2 +TC_INDEX=$3 + +source $(dirname "$0")/tc-tests-utils.sh + +cd ${PYENV_ROOT}/ && $TAR -czf $TASKCLUSTER_ARTIFACTS/pyenv.tar.gz . + +if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then + curl -sSL --fail -X PUT \ + -H "Content-Type: application/json" \ + -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ + "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" +fi; diff --git a/taskcluster/pyenv-win-amd64.yml b/taskcluster/pyenv-win-amd64.yml new file mode 100644 index 00000000..d22872b3 --- /dev/null +++ b/taskcluster/pyenv-win-amd64.yml @@ -0,0 +1,8 @@ +build: + template_file: pyenv-win-opt-base.tyml + scripts: + build: "taskcluster/pyenv-build.sh" + package: "taskcluster/pyenv-package.sh" + metadata: + name: "Builds Pyenv Windows AMD64" + description: "Setup a builds Pyenv for Windows/AMD64" diff --git a/taskcluster/pyenv-win-opt-base.tyml b/taskcluster/pyenv-win-opt-base.tyml new file mode 100644 index 00000000..b49f4499 --- /dev/null +++ b/taskcluster/pyenv-win-opt-base.tyml @@ -0,0 +1,58 @@ +$if: 'event.event in build.allowed' +then: + taskId: ${taskcluster.taskId} + provisionerId: ${taskcluster.docker.provisionerId} + workerType: ${taskcluster.docker.workerTypeWin} + taskGroupId: ${taskcluster.taskGroupId} + schedulerId: ${taskcluster.schedulerId} + created: { $fromNow: '0 sec' } + deadline: { $fromNow: '1 day' } + expires: { $fromNow: '6 months' } + scopes: + - "index:insert-task:project.deepspeech.*" + + payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + + features: + taskclusterProxy: true + + mounts: + - file: msys2-base-x86_64.tar.xz + content: + sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c + url: >- + https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download + + env: + TC_MSYS_VERSION: 'MSYS_NT-6.3' + MSYS: 'winsymlinks:nativestrict' + + command: + - >- + "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | + "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si + - .\msys64\usr\bin\bash.exe --login -cx "exit" + - .\msys64\usr\bin\bash.exe --login -cx "pacman --noconfirm -Syu" + - $let: + taskIndexExpire: { $fromNow: '6 months' } + in: > + echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C && + export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && + export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" && + export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && + echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh && + env && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${system.pyenv.win.url}`\" != \"200\" ]; then git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.win.namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k + + - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE" + + artifacts: + - type: "directory" + path: "public/" + expires: { $fromNow: '6 months' } + + metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 02c51558..4a54432f 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -5,12 +5,14 @@ set -xe export OS=$(uname) if [ "${OS}" = "Linux" ]; then export DS_ROOT_TASK=${HOME} + export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root" export SWIG_ROOT="${HOME}/ds-swig" export DS_CPU_COUNT=$(nproc) fi; if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} + export PYENV_ROOT="${TASKCLUSTER_TASK_DIR}/pyenv-root" export SWIG_ROOT="$(cygpath ${USERPROFILE})/ds-swig" export PLATFORM_EXE_SUFFIX=.exe export DS_CPU_COUNT=$(nproc) @@ -20,6 +22,7 @@ if [ "${OS}" = "Darwin" ]; then export SWIG_ROOT="${TASKCLUSTER_ORIG_TASKDIR}/ds-swig" export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} export DS_CPU_COUNT=$(sysctl hw.ncpu |cut -d' ' -f2) + export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root" export HOMEBREW_NO_AUTO_UPDATE=1 export BREW_URL=https://github.com/Homebrew/brew/tarball/2.1.14 @@ -52,6 +55,10 @@ if [ -f "${SWIG_ROOT}/bin/${DS_SWIG_BIN}" ]; then swig -swiglib fi; +PY37_OPENSSL_DIR="${PYENV_ROOT}/ssl-xenial" +export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/" +export LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH + export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts} export TASKCLUSTER_TMP_DIR=${TASKCLUSTER_TMP_DIR:-/tmp} diff --git a/taskcluster/tc-build-utils.sh b/taskcluster/tc-build-utils.sh index 0fa6b02c..0a9b633b 100755 --- a/taskcluster/tc-build-utils.sh +++ b/taskcluster/tc-build-utils.sh @@ -11,18 +11,8 @@ do_deepspeech_python_build() unset PYTHON_BIN_PATH unset PYTHONPATH - if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then - export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/DeepSpeech/.pyenv" - else - export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv" - fi; - - export PATH_WITHOUT_PYENV=${PATH} export PATH="${PYENV_ROOT}/bin:$PATH" - install_pyenv "${PYENV_ROOT}" - install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - mkdir -p wheels SETUP_FLAGS="" @@ -38,26 +28,14 @@ do_deepspeech_python_build() pyalias="${pyver}_${pyconf}" - export NUMPY_BUILD_VERSION="==1.7.0" - export NUMPY_DEP_VERSION=">=1.7.0" + maybe_numpy_min_version ${pyver} - maybe_ssl102_py37 ${pyver} - - maybe_numpy_min_version_winamd64 ${pyver} - - LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ - PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \ - pyenv_install ${pyver} ${pyalias} - - setup_pyenv_virtualenv "${pyalias}" "deepspeech" virtualenv_activate "${pyalias}" "deepspeech" python --version which pip which pip3 - # Set LD path because python ssl might require it - LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \ EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \ EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \ @@ -72,15 +50,8 @@ do_deepspeech_python_build() make -C native_client/python/ bindings-clean - unset NUMPY_BUILD_VERSION - unset NUMPY_DEP_VERSION - virtualenv_deactivate "${pyalias}" "deepspeech" done; - - # If not, and if virtualenv_deactivate does not call "pyenv uninstall ${version}" - # we get stale python2 in PATH that blocks NodeJS builds - export PATH=${PATH_WITHOUT_PYENV} } do_deepspeech_decoder_build() @@ -90,43 +61,23 @@ do_deepspeech_decoder_build() unset PYTHON_BIN_PATH unset PYTHONPATH - if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then - export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/DeepSpeech/.pyenv" - else - export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv" - fi; - export PATH="${PYENV_ROOT}/bin:$PATH" - install_pyenv "${PYENV_ROOT}" - install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - mkdir -p wheels for pyver_conf in ${SUPPORTED_PYTHON_VERSIONS}; do pyver=$(echo "${pyver_conf}" | cut -d':' -f1) pyconf=$(echo "${pyver_conf}" | cut -d':' -f2) - pyalias="${pyver}_${pyconf}" - export NUMPY_BUILD_VERSION="==1.7.0" - export NUMPY_DEP_VERSION=">=1.7.0" + maybe_numpy_min_version ${pyver} - maybe_ssl102_py37 ${pyver} - - LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ - PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \ - pyenv_install ${pyver} "${pyalias}" - - setup_pyenv_virtualenv "${pyalias}" "deepspeech" virtualenv_activate "${pyalias}" "deepspeech" python --version which pip which pip3 - # Set LD path because python ssl might require it - LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \ EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \ EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \ @@ -141,15 +92,8 @@ do_deepspeech_decoder_build() make -C native_client/ctcdecode clean-keep-third-party - unset NUMPY_BUILD_VERSION - unset NUMPY_DEP_VERSION - virtualenv_deactivate "${pyalias}" "deepspeech" done; - - # If not, and if virtualenv_deactivate does not call "pyenv uninstall ${version}" - # we get stale python2 in PATH that blocks NodeJS builds - export PATH=${PATH_WITHOUT_PYENV} } do_deepspeech_nodejs_build() diff --git a/taskcluster/tc-evaluate_tflite.sh b/taskcluster/tc-evaluate_tflite.sh index dce4b63f..bf40f6f9 100755 --- a/taskcluster/tc-evaluate_tflite.sh +++ b/taskcluster/tc-evaluate_tflite.sh @@ -9,37 +9,13 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya bitrate=$2 set_ldc_sample_filename "${bitrate}" -unset PYTHON_BIN_PATH -unset PYTHONPATH - -if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then - export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv" -else - export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv" -fi; - -export PATH="${PYENV_ROOT}/bin:$PATH" - -mkdir -p ${PYENV_ROOT} || true - download_data -install_pyenv "${PYENV_ROOT}" -install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - -maybe_ssl102_py37 ${pyver} - -maybe_numpy_min_version_winamd64 ${pyver} - -PYENV_NAME=deepspeech-test -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias} - -setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}" -virtualenv_activate "${pyalias}" "${PYENV_NAME}" +virtualenv_activate "${pyalias}" "deepspeech" deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type}) set -o pipefail -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt | cat set +o pipefail @@ -52,4 +28,4 @@ pushd ${HOME}/DeepSpeech/ds/ python evaluate_tflite.py --model "${TASKCLUSTER_TMP_DIR}/${model_name_mmap}" --scorer data/smoke_test/pruned_lm.scorer --csv data/smoke_test/ldc93s1.csv popd -virtualenv_deactivate "${pyalias}" "${PYENV_NAME}" +virtualenv_deactivate "${pyalias}" "deepspeech" diff --git a/taskcluster/tc-py-utils.sh b/taskcluster/tc-py-utils.sh index 586124bf..d3dbaaa2 100755 --- a/taskcluster/tc-py-utils.sh +++ b/taskcluster/tc-py-utils.sh @@ -61,6 +61,34 @@ install_pyenv_virtualenv() eval "$(pyenv virtualenv-init -)" } +maybe_setup_virtualenv_cross_arm() +{ + local version=$1 + local name=$2 + + if [ -z "${PYENV_ROOT}" ]; then + echo "No PYENV_ROOT set"; + exit 1; + fi; + + if [ "${OS}" != "Linux" ]; then + echo "Only for Linux"; + exit 1; + fi; + + ARCH=$(uname -m) + + if [ "${ARCH}" = "x86_64" ]; then + echo "Not for Linux/AMD64"; + exit 0; + fi; + + mkdir -p ${PYENV_ROOT}/versions/${version}/envs/ + + PIP_EXTRA_INDEX_URL="" python3 -m virtualenv -p python3 ${PYENV_ROOT}/versions/${version}/envs/${name}/ + source ${PYENV_ROOT}/versions/${version}/envs/${name}/bin/activate +} + setup_pyenv_virtualenv() { local version=$1 @@ -119,12 +147,6 @@ virtualenv_deactivate() fi; deactivate - - if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then - rm -fr ${PYENV_ROOT}/versions/${version}/ - else - pyenv uninstall --force ${name} - fi } pyenv_install() @@ -179,16 +201,11 @@ maybe_ssl102_py37() pyver=$1 unset PY37_OPENSSL - unset PY37_LDPATH - unset PY37_SOURCE_PACKAGE ARCH=$(uname -m) - case "${pyver}" in 3.7*|3.8*) if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then - PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial - if [ -d "${PY37_OPENSSL_DIR}" ]; then rm -rf "${PY37_OPENSSL_DIR}" fi @@ -210,9 +227,24 @@ maybe_ssl102_py37() ln -sfn libssl.so.1.0.0 ${PY37_OPENSSL_DIR}/usr/lib/libssl.so export PY37_OPENSSL="--with-openssl=${PY37_OPENSSL_DIR}/usr" - export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/" fi; + ;; + esac +} +maybe_numpy_min_version() +{ + local pyver=$1 + + unset NUMPY_BUILD_VERSION + unset NUMPY_DEP_VERSION + + # We set >= and < to make sure we have no numpy incompatibilities + # otherwise, `from deepspeech.impl` throws with "illegal instruction" + + ARCH=$(uname -m) + case "${OS}:${ARCH}" in + Linux:x86_64) case "${pyver}" in 3.7*) export NUMPY_BUILD_VERSION="==1.14.5" @@ -224,35 +256,48 @@ maybe_ssl102_py37() ;; esac ;; - esac -} -maybe_numpy_min_version_winamd64() -{ - local pyver=$1 + Darwin:*) + case "${pyver}" in + 3.5*|3.6) + export NUMPY_BUILD_VERSION="==1.9.0" + export NUMPY_DEP_VERSION=">=1.9.0" + ;; + 3.7*) + export NUMPY_BUILD_VERSION="==1.14.5" + export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0" + ;; + 3.8*) + export NUMPY_BUILD_VERSION="==1.17.3" + export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3" + ;; + esac + ;; - if [ "${OS}" != "${TC_MSYS_VERSION}" ]; then - return; - fi + ${TC_MSYS_VERSION}:x86_64) + case "${pyver}" in + 3.5*) + export NUMPY_BUILD_VERSION="==1.11.0" + export NUMPY_DEP_VERSION=">=1.11.0,<1.12.0" + ;; + 3.6*) + export NUMPY_BUILD_VERSION="==1.12.0" + export NUMPY_DEP_VERSION=">=1.12.0,<1.14.5" + ;; + 3.7*) + export NUMPY_BUILD_VERSION="==1.14.5" + export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0" + ;; + 3.8*) + export NUMPY_BUILD_VERSION="==1.17.3" + export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3" + ;; + esac + ;; - # We set >= and < to make sure we have no numpy incompatibilities - # otherwise, `from deepspeech.impl` throws with "illegal instruction" - case "${pyver}" in - 3.5*) - export NUMPY_BUILD_VERSION="==1.11.0" - export NUMPY_DEP_VERSION=">=1.11.0,<1.12.0" - ;; - 3.6*) - export NUMPY_BUILD_VERSION="==1.12.0" - export NUMPY_DEP_VERSION=">=1.12.0,<1.14.5" - ;; - 3.7*) - export NUMPY_BUILD_VERSION="==1.14.5" - export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0" - ;; - 3.8*) - export NUMPY_BUILD_VERSION="==1.17.3" - export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3" + *) + export NUMPY_BUILD_VERSION="==1.7.0" + export NUMPY_DEP_VERSION=">=1.7.0" ;; esac } @@ -294,8 +339,9 @@ extract_python_versions() # 3.8.x => 38 local _pyver_pkg=$(echo "${_pyver}" | cut -d'.' -f1,2 | tr -d '.') - # UCS2 => narrow unicode - # UCS4 => wide unicode + # https://www.python.org/dev/peps/pep-3149/#proposal + # 'm' => pymalloc + # 'u' => wide unicode local _py_unicode_type=$(echo "${_pyver_full}" | cut -d':' -f2) if [ "${_py_unicode_type}" = "m" ]; then local _pyconf="ucs2" diff --git a/taskcluster/tc-python-tests-prod.sh b/taskcluster/tc-python-tests-prod.sh index 2b5fe848..8e0f18ad 100644 --- a/taskcluster/tc-python-tests-prod.sh +++ b/taskcluster/tc-python-tests-prod.sh @@ -9,19 +9,6 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya bitrate=$2 set_ldc_sample_filename "${bitrate}" -unset PYTHON_BIN_PATH -unset PYTHONPATH - -if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then - export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv" -else - export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv" -fi; - -export PATH="${PYENV_ROOT}/bin:$PATH" - -mkdir -p ${PYENV_ROOT} || true - model_source=${DEEPSPEECH_PROD_MODEL} model_name=$(basename "${model_source}") @@ -30,24 +17,13 @@ model_name_mmap=$(basename "${model_source_mmap}") download_data -install_pyenv "${PYENV_ROOT}" -install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - -maybe_ssl102_py37 ${pyver} - -maybe_numpy_min_version_winamd64 ${pyver} - -PYENV_NAME=deepspeech-test -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias} - -setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}" -virtualenv_activate "${pyalias}" "${PYENV_NAME}" +virtualenv_activate "${pyalias}" "deepspeech" deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type}) -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat run_prod_inference_tests "${bitrate}" run_prod_concurrent_stream_tests "${bitrate}" -virtualenv_deactivate "${pyalias}" "${PYENV_NAME}" +virtualenv_deactivate "${pyalias}" "deepspeech" diff --git a/taskcluster/tc-python-tests.sh b/taskcluster/tc-python-tests.sh index 6f101389..92a2b792 100644 --- a/taskcluster/tc-python-tests.sh +++ b/taskcluster/tc-python-tests.sh @@ -9,40 +9,16 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya bitrate=$2 set_ldc_sample_filename "${bitrate}" -unset PYTHON_BIN_PATH -unset PYTHONPATH - -if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then - export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv" -else - export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv" -fi; - -export PATH="${PYENV_ROOT}/bin:$PATH" - -mkdir -p ${PYENV_ROOT} || true - download_data -install_pyenv "${PYENV_ROOT}" -install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - -maybe_ssl102_py37 ${pyver} - -maybe_numpy_min_version_winamd64 ${pyver} - -PYENV_NAME=deepspeech-test -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias} - -setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}" -virtualenv_activate "${pyalias}" "${PYENV_NAME}" +virtualenv_activate "${pyalias}" "deepspeech" deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type}) -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat which deepspeech deepspeech --version run_all_inference_tests -virtualenv_deactivate "${pyalias}" "${PYENV_NAME}" +virtualenv_deactivate "${pyalias}" "deepspeech" diff --git a/taskcluster/tc-python_tflite-tests-prod.sh b/taskcluster/tc-python_tflite-tests-prod.sh index 00a0615c..5b2df0ec 100644 --- a/taskcluster/tc-python_tflite-tests-prod.sh +++ b/taskcluster/tc-python_tflite-tests-prod.sh @@ -9,19 +9,6 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya bitrate=$2 set_ldc_sample_filename "${bitrate}" -unset PYTHON_BIN_PATH -unset PYTHONPATH - -if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then - export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv" -else - export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv" -fi; - -export PATH="${PYENV_ROOT}/bin:$PATH" - -mkdir -p ${PYENV_ROOT} || true - model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") model_name_mmap=$(basename "${model_source}") @@ -30,25 +17,16 @@ export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT} download_data -install_pyenv "${PYENV_ROOT}" -install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" +maybe_setup_virtualenv_cross_arm "${pyalias}" "deepspeech" -maybe_ssl102_py37 ${pyver} - -maybe_numpy_min_version_winamd64 ${pyver} - -PYENV_NAME=deepspeech-test -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias} - -setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}" -virtualenv_activate "${pyalias}" "${PYENV_NAME}" +virtualenv_activate "${pyalias}" "deepspeech" deepspeech_pkg_url=$(get_python_pkg_url "${pyver_pkg}" "${py_unicode_type}" "deepspeech_tflite") -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat which deepspeech deepspeech --version run_prodtflite_inference_tests "${bitrate}" -virtualenv_deactivate "${pyalias}" "${PYENV_NAME}" +virtualenv_deactivate "${pyalias}" "deepspeech" diff --git a/taskcluster/tc-python_tflite-tests.sh b/taskcluster/tc-python_tflite-tests.sh index b2759aa6..1deb1253 100644 --- a/taskcluster/tc-python_tflite-tests.sh +++ b/taskcluster/tc-python_tflite-tests.sh @@ -9,44 +9,22 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya bitrate=$2 set_ldc_sample_filename "${bitrate}" -unset PYTHON_BIN_PATH -unset PYTHONPATH - -if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then - export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv" -else - export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv" -fi; - -export PATH="${PYENV_ROOT}/bin:$PATH" - -mkdir -p ${PYENV_ROOT} || true - model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite} model_name=$(basename "${model_source}") model_name_mmap=$(basename "${model_source}") download_data -install_pyenv "${PYENV_ROOT}" -install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" +maybe_setup_virtualenv_cross_arm "${pyalias}" "deepspeech" -maybe_ssl102_py37 ${pyver} - -maybe_numpy_min_version_winamd64 ${pyver} - -PYENV_NAME=deepspeech-test -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias} - -setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}" -virtualenv_activate "${pyalias}" "${PYENV_NAME}" +virtualenv_activate "${pyalias}" "deepspeech" deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type}) -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat which deepspeech deepspeech --version run_all_inference_tests -virtualenv_deactivate "${pyalias}" "${PYENV_NAME}" +virtualenv_deactivate "${pyalias}" "deepspeech" diff --git a/taskcluster/tc-single-shot-inference.sh b/taskcluster/tc-single-shot-inference.sh index 24abe19d..f10ea106 100755 --- a/taskcluster/tc-single-shot-inference.sh +++ b/taskcluster/tc-single-shot-inference.sh @@ -9,24 +9,11 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya ds=$2 frozen=$2 -unset PYTHON_BIN_PATH -unset PYTHONPATH -export PYENV_ROOT="${HOME}/ds-train/.pyenv" -export PATH="${PYENV_ROOT}/bin:${HOME}/bin:$PATH" - -mkdir -p ${PYENV_ROOT} || true mkdir -p ${TASKCLUSTER_ARTIFACTS} || true mkdir -p /tmp/train || true mkdir -p /tmp/train_tflite || true -install_pyenv "${PYENV_ROOT}" -install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - -PYENV_NAME=deepspeech-train -PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv_install ${pyver} - -setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}" -virtualenv_activate "${pyver}" "${PYENV_NAME}" +virtualenv_activate "${pyalias}" "deepspeech" set -o pipefail pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat @@ -34,10 +21,10 @@ pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt | cat set +o pipefail decoder_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type} "ds_ctcdecoder" "${DECODER_ARTIFACTS_ROOT}") -LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${decoder_pkg_url} | cat +LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${decoder_pkg_url} | cat pushd ${HOME}/DeepSpeech/ds/ time ./bin/run-tc-ldc93s1_singleshotinference.sh popd -virtualenv_deactivate "${pyver}" "${PYENV_NAME}" +virtualenv_deactivate "${pyver}" "deepspeech" diff --git a/taskcluster/tc-train-tests.sh b/taskcluster/tc-train-tests.sh index 47bacfa8..1be6533b 100644 --- a/taskcluster/tc-train-tests.sh +++ b/taskcluster/tc-train-tests.sh @@ -4,43 +4,15 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -pyver_full=$1 +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" + bitrate=$2 -if [ -z "${pyver_full}" ]; then - echo "No python version given, aborting." - exit 1 -fi; - -pyver=$(echo "${pyver_full}" | cut -d':' -f1) - -# 2.7.x => 27 -pyver_pkg=$(echo "${pyver}" | cut -d'.' -f1,2 | tr -d '.') - -py_unicode_type=$(echo "${pyver_full}" | cut -d':' -f2) -if [ "${py_unicode_type}" = "m" ]; then - pyconf="ucs2" -elif [ "${py_unicode_type}" = "mu" ]; then - pyconf="ucs4" -fi; - -unset PYTHON_BIN_PATH -unset PYTHONPATH -export PYENV_ROOT="${HOME}/ds-train/.pyenv" -export PATH="${PYENV_ROOT}/bin:${HOME}/bin:$PATH" - -mkdir -p ${PYENV_ROOT} || true mkdir -p ${TASKCLUSTER_ARTIFACTS} || true mkdir -p /tmp/train || true mkdir -p /tmp/train_tflite || true -install_pyenv "${PYENV_ROOT}" -install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - -PYENV_NAME=deepspeech-train -PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver} -pyenv virtualenv ${pyver} ${PYENV_NAME} -source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate +virtualenv_activate "${pyalias}" "deepspeech" set -o pipefail pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat @@ -51,12 +23,7 @@ pushd ${HOME}/DeepSpeech/ds/ verify_ctcdecoder_url popd -platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') -whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')" -decoder_pkg="ds_ctcdecoder-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" - -decoder_pkg_url=${DECODER_ARTIFACTS_ROOT}/${decoder_pkg} - +decoder_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type} "ds_ctcdecoder" "${DECODER_ARTIFACTS_ROOT}") LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} ${decoder_pkg_url} | cat # Prepare correct arguments for training @@ -97,4 +64,4 @@ pushd ${HOME}/DeepSpeech/ds/ time ./bin/run-tc-ldc93s1_checkpoint.sh popd -deactivate +virtualenv_deactivate "${pyalias}" "deepspeech" diff --git a/taskcluster/tc-transfer-tests.sh b/taskcluster/tc-transfer-tests.sh index f0a963a6..594c804a 100644 --- a/taskcluster/tc-transfer-tests.sh +++ b/taskcluster/tc-transfer-tests.sh @@ -4,42 +4,13 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -pyver_full=$1 +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" -if [ -z "${pyver_full}" ]; then - echo "No python version given, aborting." - exit 1 -fi; - -pyver=$(echo "${pyver_full}" | cut -d':' -f1) - -# 2.7.x => 27 -pyver_pkg=$(echo "${pyver}" | cut -d'.' -f1,2 | tr -d '.') - -py_unicode_type=$(echo "${pyver_full}" | cut -d':' -f2) -if [ "${py_unicode_type}" = "m" ]; then - pyconf="ucs2" -elif [ "${py_unicode_type}" = "mu" ]; then - pyconf="ucs4" -fi; - -unset PYTHON_BIN_PATH -unset PYTHONPATH -export PYENV_ROOT="${HOME}/ds-train/.pyenv" -export PATH="${PYENV_ROOT}/bin:${HOME}/bin:$PATH" - -mkdir -p ${PYENV_ROOT} || true mkdir -p ${TASKCLUSTER_ARTIFACTS} || true mkdir -p /tmp/train || true mkdir -p /tmp/train_tflite || true -install_pyenv "${PYENV_ROOT}" -install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv" - -PYENV_NAME=deepspeech-train -PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver} -pyenv virtualenv ${pyver} ${PYENV_NAME} -source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate +virtualenv_activate "${pyalias}" "deepspeech" set -o pipefail pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat @@ -50,16 +21,11 @@ pushd ${HOME}/DeepSpeech/ds/ verify_ctcdecoder_url popd -platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));') -whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')" -decoder_pkg="ds_ctcdecoder-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl" - -decoder_pkg_url=${DECODER_ARTIFACTS_ROOT}/${decoder_pkg} - +decoder_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type} "ds_ctcdecoder" "${DECODER_ARTIFACTS_ROOT}") LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} ${decoder_pkg_url} | cat pushd ${HOME}/DeepSpeech/ds/ time ./bin/run-tc-transfer.sh popd -deactivate +virtualenv_deactivate "${pyalias}" "deepspeech" diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index 9c69ac37..a11f3d8f 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -45,8 +45,8 @@ then: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && - export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && cd ../tc-workdir/ && - (mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) && + export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && + (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && export TASKCLUSTER_TASK_DIR="$(pwd)" && export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" && export LC_ALL=C && @@ -55,20 +55,21 @@ then: export PIP_DEFAULT_TIMEOUT=60 && env && trap "export TASKCLUSTER_TASK_EXIT_CODE=$? && - (mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ || true) && cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ && exit $TASKCLUSTER_TASK_EXIT_CODE" 0 && git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && cd $TASKCLUSTER_TASK_DIR && + (mkdir pyenv-root/ && gtar -C pyenv-root/ -xf $TASKCLUSTER_ORIG_TASKDIR/pyenv.tar.gz) && + (mkdir homebrew-tests/ && gtar -C homebrew-tests/ -xf $TASKCLUSTER_ORIG_TASKDIR/homebrew-tests.tar.gz) && ${extraSystemSetup} && /bin/bash ${build.args.tests_cmdline} mounts: - - cacheName: deepspeech-macos-pyenv - directory: pyenv.cache/ - - directory: ../tc-workdir/homebrew-tests - format: tar.gz + - file: pyenv.tar.gz + content: + url: ${system.pyenv.osx.url} + - file: homebrew-tests.tar.gz content: url: ${system.homebrew_tests.url} diff --git a/taskcluster/test-evaluate_tflite-linux-amd64-py36m-opt.yml b/taskcluster/test-evaluate_tflite-linux-amd64-py36m-opt.yml index 513fd28f..3356866e 100644 --- a/taskcluster/test-evaluate_tflite-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-evaluate_tflite-linux-amd64-py36m-opt.yml @@ -4,11 +4,8 @@ build: - "linux-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-evaluate_tflite.sh 3.6.4:m 16k" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-evaluate_tflite.sh 3.6.10:m 16k" metadata: name: "DeepSpeech Linux AMD64 CPU evaluate_tflite.py Py3.6 (16kHz)" description: "Test evaluate_tflite.py on Linux/AMD64 using upstream TensorFlow Python 3.6, CPU only, optimized version" diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 9686d313..a8cde72b 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -48,7 +48,7 @@ then: apt-get -qq update && apt-get -qq -y install curl python-simplejson git pixz sox sudo wget && ${extraSystemSetup} && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux} && - echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}&& mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline} diff --git a/taskcluster/test-python_35_16k-linux-amd64-opt.yml b/taskcluster/test-python_35_16k-linux-amd64-opt.yml index 62d63d62..c47d90fa 100644 --- a/taskcluster/test-python_35_16k-linux-amd64-opt.yml +++ b/taskcluster/test-python_35_16k-linux-amd64-opt.yml @@ -4,9 +4,6 @@ build: - "linux-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.8:m 16k" metadata: diff --git a/taskcluster/test-python_35_16k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_35_16k-linux-amd64-prod_pbmodel-opt.yml index 07d1ba3b..808220f4 100644 --- a/taskcluster/test-python_35_16k-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_35_16k-linux-amd64-prod_pbmodel-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-cpu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.5.8:m 16k" metadata: diff --git a/taskcluster/test-python_35_8k-linux-amd64-opt.yml b/taskcluster/test-python_35_8k-linux-amd64-opt.yml index 144ad3fa..5d02920a 100644 --- a/taskcluster/test-python_35_8k-linux-amd64-opt.yml +++ b/taskcluster/test-python_35_8k-linux-amd64-opt.yml @@ -4,9 +4,6 @@ build: - "linux-amd64-cpu-opt" - "test-training_8k-linux-amd64-py36m-opt" test_model_task: "test-training_8k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.8:m 8k" metadata: diff --git a/taskcluster/test-python_35_8k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_35_8k-linux-amd64-prod_pbmodel-opt.yml index 80eaaddc..0539c5c3 100644 --- a/taskcluster/test-python_35_8k-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_35_8k-linux-amd64-prod_pbmodel-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-cpu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.5.8:m 8k" metadata: diff --git a/taskcluster/test-python_35_tflite_16k-linux-amd64-prod-opt.yml b/taskcluster/test-python_35_tflite_16k-linux-amd64-prod-opt.yml index ddcf4ebc..09834d94 100644 --- a/taskcluster/test-python_35_tflite_16k-linux-amd64-prod-opt.yml +++ b/taskcluster/test-python_35_tflite_16k-linux-amd64-prod-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-tflite-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.5.8:m 16k" metadata: diff --git a/taskcluster/test-python_35_tflite_8k-linux-amd64-prod-opt.yml b/taskcluster/test-python_35_tflite_8k-linux-amd64-prod-opt.yml index 1e99ea6e..f6040db2 100644 --- a/taskcluster/test-python_35_tflite_8k-linux-amd64-prod-opt.yml +++ b/taskcluster/test-python_35_tflite_8k-linux-amd64-prod-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-tflite-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.5.8:m 8k" metadata: diff --git a/taskcluster/test-python_36_16k-linux-amd64-opt.yml b/taskcluster/test-python_36_16k-linux-amd64-opt.yml index f85cc112..e2ad438d 100644 --- a/taskcluster/test-python_36_16k-linux-amd64-opt.yml +++ b/taskcluster/test-python_36_16k-linux-amd64-opt.yml @@ -4,9 +4,6 @@ build: - "linux-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.6.10:m 16k" metadata: diff --git a/taskcluster/test-python_36_16k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_36_16k-linux-amd64-prod_pbmodel-opt.yml index 705f4609..ad4b4ab9 100644 --- a/taskcluster/test-python_36_16k-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_36_16k-linux-amd64-prod_pbmodel-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-cpu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.6.10:m 16k" metadata: diff --git a/taskcluster/test-python_36_8k-linux-amd64-opt.yml b/taskcluster/test-python_36_8k-linux-amd64-opt.yml index 6070007f..c0a40f02 100644 --- a/taskcluster/test-python_36_8k-linux-amd64-opt.yml +++ b/taskcluster/test-python_36_8k-linux-amd64-opt.yml @@ -4,9 +4,6 @@ build: - "linux-amd64-cpu-opt" - "test-training_8k-linux-amd64-py36m-opt" test_model_task: "test-training_8k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.6.10:m 8k" metadata: diff --git a/taskcluster/test-python_36_8k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_36_8k-linux-amd64-prod_pbmodel-opt.yml index ce321971..a5e2e21a 100644 --- a/taskcluster/test-python_36_8k-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_36_8k-linux-amd64-prod_pbmodel-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-cpu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.6.10:m 8k" metadata: diff --git a/taskcluster/test-python_36_tflite_16k-linux-amd64-prod-opt.yml b/taskcluster/test-python_36_tflite_16k-linux-amd64-prod-opt.yml index b4945f30..ea3f1f90 100644 --- a/taskcluster/test-python_36_tflite_16k-linux-amd64-prod-opt.yml +++ b/taskcluster/test-python_36_tflite_16k-linux-amd64-prod-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-tflite-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.6.10:m 16k" metadata: diff --git a/taskcluster/test-python_36_tflite_8k-linux-amd64-prod-opt.yml b/taskcluster/test-python_36_tflite_8k-linux-amd64-prod-opt.yml index 2159704e..65aa60ea 100644 --- a/taskcluster/test-python_36_tflite_8k-linux-amd64-prod-opt.yml +++ b/taskcluster/test-python_36_tflite_8k-linux-amd64-prod-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-tflite-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.6.10:m 8k" metadata: diff --git a/taskcluster/test-python_37_16k-linux-amd64-opt.yml b/taskcluster/test-python_37_16k-linux-amd64-opt.yml index db2bc14a..738e5241 100644 --- a/taskcluster/test-python_37_16k-linux-amd64-opt.yml +++ b/taskcluster/test-python_37_16k-linux-amd64-opt.yml @@ -4,9 +4,6 @@ build: - "linux-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.6:m 16k" metadata: diff --git a/taskcluster/test-python_37_16k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_37_16k-linux-amd64-prod_pbmodel-opt.yml index 2d2e4ff3..4f3df1f3 100644 --- a/taskcluster/test-python_37_16k-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_37_16k-linux-amd64-prod_pbmodel-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-cpu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.7.6:m 16k" metadata: diff --git a/taskcluster/test-python_37_8k-linux-amd64-opt.yml b/taskcluster/test-python_37_8k-linux-amd64-opt.yml index 3aa13e5c..3d16b4f3 100644 --- a/taskcluster/test-python_37_8k-linux-amd64-opt.yml +++ b/taskcluster/test-python_37_8k-linux-amd64-opt.yml @@ -4,9 +4,6 @@ build: - "linux-amd64-cpu-opt" - "test-training_8k-linux-amd64-py36m-opt" test_model_task: "test-training_8k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.6:m 8k" metadata: diff --git a/taskcluster/test-python_37_8k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_37_8k-linux-amd64-prod_pbmodel-opt.yml index 559b7292..b515cfd8 100644 --- a/taskcluster/test-python_37_8k-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_37_8k-linux-amd64-prod_pbmodel-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-cpu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.7.6:m 8k" metadata: diff --git a/taskcluster/test-python_37_tflite_16k-linux-amd64-prod-opt.yml b/taskcluster/test-python_37_tflite_16k-linux-amd64-prod-opt.yml index 594a4190..dec375c7 100644 --- a/taskcluster/test-python_37_tflite_16k-linux-amd64-prod-opt.yml +++ b/taskcluster/test-python_37_tflite_16k-linux-amd64-prod-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-tflite-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.7.6:m 16k" metadata: diff --git a/taskcluster/test-python_37_tflite_8k-linux-amd64-prod-opt.yml b/taskcluster/test-python_37_tflite_8k-linux-amd64-prod-opt.yml index 02e9be6b..2faeb7e3 100644 --- a/taskcluster/test-python_37_tflite_8k-linux-amd64-prod-opt.yml +++ b/taskcluster/test-python_37_tflite_8k-linux-amd64-prod-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-tflite-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.7.6:m 8k" metadata: diff --git a/taskcluster/test-python_38_16k-linux-amd64-opt.yml b/taskcluster/test-python_38_16k-linux-amd64-opt.yml index dc8fb55a..0828219f 100644 --- a/taskcluster/test-python_38_16k-linux-amd64-opt.yml +++ b/taskcluster/test-python_38_16k-linux-amd64-opt.yml @@ -4,9 +4,6 @@ build: - "linux-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.1: 16k" metadata: diff --git a/taskcluster/test-python_38_16k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_38_16k-linux-amd64-prod_pbmodel-opt.yml index 65a8dc9e..e76f5929 100644 --- a/taskcluster/test-python_38_16k-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_38_16k-linux-amd64-prod_pbmodel-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-cpu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.8.1: 16k" metadata: diff --git a/taskcluster/test-python_38_8k-linux-amd64-opt.yml b/taskcluster/test-python_38_8k-linux-amd64-opt.yml index e7f000c8..402fcc78 100644 --- a/taskcluster/test-python_38_8k-linux-amd64-opt.yml +++ b/taskcluster/test-python_38_8k-linux-amd64-opt.yml @@ -4,9 +4,6 @@ build: - "linux-amd64-cpu-opt" - "test-training_8k-linux-amd64-py36m-opt" test_model_task: "test-training_8k-linux-amd64-py36m-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.1: 8k" metadata: diff --git a/taskcluster/test-python_38_8k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-python_38_8k-linux-amd64-prod_pbmodel-opt.yml index 4982f46f..72d0ff47 100644 --- a/taskcluster/test-python_38_8k-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-python_38_8k-linux-amd64-prod_pbmodel-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-cpu-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.8.1: 8k" metadata: diff --git a/taskcluster/test-python_38_tflite_16k-linux-amd64-prod-opt.yml b/taskcluster/test-python_38_tflite_16k-linux-amd64-prod-opt.yml index 35955a8b..a595680d 100644 --- a/taskcluster/test-python_38_tflite_16k-linux-amd64-prod-opt.yml +++ b/taskcluster/test-python_38_tflite_16k-linux-amd64-prod-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-tflite-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.8.1: 16k" metadata: diff --git a/taskcluster/test-python_38_tflite_8k-linux-amd64-prod-opt.yml b/taskcluster/test-python_38_tflite_8k-linux-amd64-prod-opt.yml index dc795f8f..bf06a75a 100644 --- a/taskcluster/test-python_38_tflite_8k-linux-amd64-prod-opt.yml +++ b/taskcluster/test-python_38_tflite_8k-linux-amd64-prod-opt.yml @@ -2,9 +2,6 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-tflite-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.8.1: 8k" metadata: diff --git a/taskcluster/test-singleshotinference-linux-amd64-py36m-opt.yml b/taskcluster/test-singleshotinference-linux-amd64-py36m-opt.yml index 987b4f8e..610e0c8f 100644 --- a/taskcluster/test-singleshotinference-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-singleshotinference-linux-amd64-py36m-opt.yml @@ -2,11 +2,8 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-ctc-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-single-shot-inference.sh 3.6.4:m" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-single-shot-inference.sh 3.6.10:m" metadata: name: "DeepSpeech Linux AMD64 CPU single-shot inference Py3.6" description: "Single-shot inference a DeepSpeech LDC93S1 checkpoint for Linux/AMD64 using upstream TensorFlow Python 3.6, CPU only, optimized version" diff --git a/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml b/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml index 2af536ef..e950969f 100644 --- a/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml +++ b/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml @@ -2,11 +2,8 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-ctc-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.5.5:m 16k" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.5.8:m 16k" metadata: name: "DeepSpeech Linux AMD64 CPU 16kHz training Py3.5" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 16kHz Python 3.5, CPU only, optimized version" diff --git a/taskcluster/test-training_16k-linux-amd64-py36m-opt.yml b/taskcluster/test-training_16k-linux-amd64-py36m-opt.yml index 36a50e05..0bb84191 100644 --- a/taskcluster/test-training_16k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training_16k-linux-amd64-py36m-opt.yml @@ -2,11 +2,8 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-ctc-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.4:m 16k" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 16k" metadata: name: "DeepSpeech Linux AMD64 CPU 16kHz training Py3.6" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 16kHz Python 3.6, CPU only, optimized version" diff --git a/taskcluster/test-training_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training_8k-linux-amd64-py36m-opt.yml index 32f9b8f4..e4164a9b 100644 --- a/taskcluster/test-training_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training_8k-linux-amd64-py36m-opt.yml @@ -2,11 +2,8 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-ctc-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.4:m 8k" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 8k" metadata: name: "DeepSpeech Linux AMD64 CPU 8kHz training Py3.6" description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 8kHz Python 3.6, CPU only, optimized version" diff --git a/taskcluster/test-transfer-linux-amd64-py36m-opt.yml b/taskcluster/test-transfer-linux-amd64-py36m-opt.yml index dc084c98..e94101ea 100644 --- a/taskcluster/test-transfer-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-transfer-linux-amd64-py36m-opt.yml @@ -2,11 +2,8 @@ build: template_file: test-linux-opt-base.tyml dependencies: - "linux-amd64-ctc-opt" - system_setup: - > - apt-get -qq -y install ${python.packages_trusty.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-transfer-tests.sh 3.6.4:m" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-transfer-tests.sh 3.6.10:m" metadata: name: "DeepSpeech Linux AMD64 CPU transfer learning Py3.6" description: "Training a DeepSpeech LDC93S1 model with transfer learning for Linux/AMD64 16kHz Python 3.6, CPU only, optimized version" diff --git a/taskcluster/test-win-opt-base.tyml b/taskcluster/test-win-opt-base.tyml index 783c2cc4..72a52702 100644 --- a/taskcluster/test-win-opt-base.tyml +++ b/taskcluster/test-win-opt-base.tyml @@ -19,12 +19,6 @@ then: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - mounts: - - file: msys2-base-x86_64.tar.xz - content: - sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c - url: >- - https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download env: $let: @@ -63,6 +57,7 @@ then: git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && cd $TASKCLUSTER_TASK_DIR && + (mkdir pyenv-root/ && 7z x -so $USERPROFILE/pyenv.tar.gz | 7z x -opyenv-root/ -aoa -ttar -si ) && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar vim && ${extraSystemSetup} && @@ -70,6 +65,16 @@ then: export TASKCLUSTER_TASK_EXIT_CODE=$? && cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" + mounts: + - file: msys2-base-x86_64.tar.xz + content: + sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c + url: >- + https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download + - file: pyenv.tar.gz + content: + url: ${system.pyenv.win.url} + metadata: name: ${build.metadata.name} description: ${build.metadata.description} diff --git a/taskcluster/win-amd64-cpu-opt.yml b/taskcluster/win-amd64-cpu-opt.yml index 1b7e96fa..32512094 100644 --- a/taskcluster/win-amd64-cpu-opt.yml +++ b/taskcluster/win-amd64-cpu-opt.yml @@ -7,6 +7,7 @@ build: dependencies: - "node-gyp-cache" - "swig-win-amd64" + - "pyenv-win-amd64" tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh" diff --git a/taskcluster/win-amd64-gpu-opt.yml b/taskcluster/win-amd64-gpu-opt.yml index 236d6524..077e79db 100644 --- a/taskcluster/win-amd64-gpu-opt.yml +++ b/taskcluster/win-amd64-gpu-opt.yml @@ -7,6 +7,7 @@ build: dependencies: - "node-gyp-cache" - "swig-win-amd64" + - "pyenv-win-amd64" tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win-cuda/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh --cuda" diff --git a/taskcluster/win-amd64-tflite-opt.yml b/taskcluster/win-amd64-tflite-opt.yml index bd077dcf..993f04f3 100644 --- a/taskcluster/win-amd64-tflite-opt.yml +++ b/taskcluster/win-amd64-tflite-opt.yml @@ -7,6 +7,7 @@ build: dependencies: - "node-gyp-cache" - "swig-win-amd64" + - "pyenv-win-amd64" tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh --tflite" diff --git a/taskcluster/win-opt-base.tyml b/taskcluster/win-opt-base.tyml index ab4076c5..8f8acd40 100644 --- a/taskcluster/win-opt-base.tyml +++ b/taskcluster/win-opt-base.tyml @@ -28,25 +28,10 @@ routes: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - mounts: - - file: msys2-base-x86_64.tar.xz - content: - sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c - url: >- - https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download - - directory: .node-gyp - format: tar.gz - content: - url: ${system.node_gyp_cache.url} - - directory: ds-swig - format: tar.gz - content: - url: ${system.swig.cache.win_amd64} env: TC_MSYS_VERSION: 'MSYS_NT-6.3' MSYS: 'winsymlinks:nativestrict' - TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} EXAMPLES_CLONE_URL: "https://github.com/mozilla/DeepSpeech-examples" EXAMPLES_CHECKOUT_TARGET: "master" @@ -64,13 +49,14 @@ payload: (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR && env && ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64 && - (wget -O - $TENSORFLOW_BUILD_ARTIFACT | 7z x -txz -si -so | 7z x -aoa -ttar -si ) && + (7z x -txz -so $USERPROFILE/home.tar.xz | 7z x -aoa -ttar -si ) && git clone --quiet $EXAMPLES_CLONE_URL $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && git checkout --quiet $EXAMPLES_CHECKOUT_TARGET && git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client && cd $TASKCLUSTER_TASK_DIR && + (mkdir pyenv-root/ && 7z x -so $USERPROFILE/pyenv.tar.gz | 7z x -opyenv-root/ -aoa -ttar -si ) && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar make && pacman --noconfirm -S zip && @@ -83,6 +69,27 @@ payload: - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE" + mounts: + - file: msys2-base-x86_64.tar.xz + content: + sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c + url: >- + https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download + - file: home.tar.xz + content: + url: ${build.tensorflow} + - file: pyenv.tar.gz + content: + url: ${system.pyenv.win.url} + - directory: .node-gyp + format: tar.gz + content: + url: ${system.node_gyp_cache.url} + - directory: ds-swig + format: tar.gz + content: + url: ${system.swig.cache.win_amd64} + artifacts: - type: "directory" path: "public/"