experiments
/
STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Produce pyenv ready-to-use

This commit is contained in:
Alexandre Lissy 2020-02-20 13:20:26 +01:00
parent 46e7993075
commit 8029f3d7dd
77 changed files with 563 additions and 529 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -13,7 +13,7 @@ semver
setuptools
# Requirements for transcribe.py
webrtcvad
webrtcvad[transcribe]
# Requirements for importers
sox

15
taskcluster/.shared.yml
View File

@ -2,9 +2,12 @@ python:
packages_trusty:
apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libpng-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev'
packages_buster:
apt: 'make build-essential libssl-dev libffi-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
apt: 'python3-virtualenv python3-setuptools python3-pip python3-wheel python3-pkg-resources'
packages_docs_bionic:
apt: 'python3 python3-pip zip doxygen'
tensorflow:
packages_trusty:
apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python zlib1g-dev'
electronjs:
packages_xenial:
apt: 'libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xvfb'
@ -52,6 +55,16 @@ system:
homebrew_tests:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz'
namespace: 'project.deepspeech.homebrew_tests.4'
pyenv:
linux:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.5/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.linux.5'
osx:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.osx.5/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.osx.5'
win:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.5/artifacts/public/pyenv.tar.gz'
namespace: 'project.deepspeech.pyenv.win.5'
swig:
repo: "https://github.com/lissyx/swig"
sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118"

1
taskcluster/android-arm64-cpu-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64"

1
taskcluster/android-armv7-cpu-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7"

14
taskcluster/android-java-opt.yml
View File

@ -1,16 +1,20 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "android-arm64-cpu-opt"
- "android-armv7-cpu-opt"
- "android-x86_64-cpu-opt"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-apk"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-apk"
- "index.project.deepspeech.deepspeech.native_client.android-apk.${event.head.sha}"
dependencies:
- "android-arm64-cpu-opt"
- "android-armv7-cpu-opt"
- "android-x86_64-cpu-opt"
system_setup:
>
apt-get -qq -y install curl
apt-get -qq -y install curl software-properties-common &&
add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update &&
DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk &&
java -version && update-ca-certificates -f
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.android-armv7/artifacts/public/home.tar.xz"
scripts:
build: "taskcluster/android-apk-build.sh"

1
taskcluster/android-x86_64-cpu-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64"

1
taskcluster/darwin-amd64-cpu-opt.yml
View File

@ -4,6 +4,7 @@ build:
- "swig-darwin-amd64"
- "node-gyp-cache"
- "homebrew_builds-darwin-amd64"
- "pyenv-darwin-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx"

1
taskcluster/darwin-amd64-ctc-opt.yml
View File

@ -4,6 +4,7 @@ build:
- "swig-darwin-amd64"
- "node-gyp-cache"
- "homebrew_builds-darwin-amd64"
- "pyenv-darwin-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc"

1
taskcluster/darwin-amd64-tflite-opt.yml
View File

@ -4,6 +4,7 @@ build:
- "swig-darwin-amd64"
- "node-gyp-cache"
- "homebrew_builds-darwin-amd64"
- "pyenv-darwin-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite"

64
taskcluster/darwin-opt-base.tyml
View File

@ -26,11 +26,6 @@ routes:
then:
{ $eval: build.routes }
scopes:
$if: '(event.event != "push") && (event.event != "tag")'
then:
- generic-worker:cache:deepspeech-macos-pyenv
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
@ -53,8 +48,8 @@ payload:
- "-cxe"
- >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export LC_ALL=C &&
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
@ -62,7 +57,6 @@ payload:
export HOMEBREW_NO_AUTO_UPDATE=1 &&
env &&
trap "export TASKCLUSTER_TASK_EXIT_CODE=$? &&
(mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ || true) &&
cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ &&
exit $TASKCLUSTER_TASK_EXIT_CODE" 0 &&
(pixz -d < $TASKCLUSTER_ORIG_TASKDIR/home.tar.xz | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
@ -70,6 +64,8 @@ payload:
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
cd $TASKCLUSTER_TASK_DIR &&
(mkdir pyenv-root/ && gtar -C pyenv-root/ -xf $TASKCLUSTER_ORIG_TASKDIR/pyenv.tar.gz) &&
(mkdir homebrew-builds/ && gtar -C homebrew-builds/ -xf $TASKCLUSTER_ORIG_TASKDIR/homebrew-builds.tar.gz) &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package}
@ -82,41 +78,23 @@ payload:
else: { $fromNow: '7 days' }
mounts:
$if: '(event.event != "push") && (event.event != "tag")'
then:
- cacheName: deepspeech-macos-pyenv
directory: pyenv.cache/
- directory: ${system.homedir.osx}/Library/Caches/node-gyp/
format: tar.gz
content:
url: ${system.node_gyp_cache.url}
- directory: ds-swig
format: tar.gz
content:
url: ${system.swig.cache.darwin_amd64}
- file: home.tar.xz
content:
url: ${build.tensorflow}
- directory: ../tc-workdir/homebrew-builds
format: tar.gz
content:
url: ${system.homebrew_builds.url}
else:
- directory: ${system.homedir.osx}/Library/Caches/node-gyp/
format: tar.gz
content:
url: ${system.node_gyp_cache.url}
- directory: ds-swig
format: tar.gz
content:
url: ${system.swig.cache.darwin_amd64}
- file: home.tar.xz
content:
url: ${build.tensorflow}
- directory: ../tc-workdir/homebrew-builds
format: tar.gz
content:
url: ${system.homebrew_builds.url}
- file: pyenv.tar.gz
content:
url: ${system.pyenv.osx.url}
- file: homebrew-builds.tar.gz
content:
url: ${system.homebrew_builds.url}
- directory: ${system.homedir.osx}/Library/Caches/node-gyp/
format: tar.gz
content:
url: ${system.node_gyp_cache.url}
- directory: ds-swig
format: tar.gz
content:
url: ${system.swig.cache.darwin_amd64}
- file: home.tar.xz
content:
url: ${build.tensorflow}
metadata:
name: ${build.metadata.name}

4
taskcluster/linux-amd64-cpu-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu"
@ -10,8 +11,7 @@ build:
system_setup:
>
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz"
scripts:
build: "taskcluster/host-build.sh"

4
taskcluster/linux-amd64-ctc-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-ctc"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-ctc"
@ -10,8 +11,7 @@ build:
system_setup:
>
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz"
scripts:
build: 'taskcluster/decoder-build.sh'

4
taskcluster/linux-amd64-gpu-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu"
@ -10,8 +11,7 @@ build:
system_setup:
>
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.gpu/artifacts/public/home.tar.xz"
maxRunTime: 14400
scripts:

4
taskcluster/linux-amd64-tflite-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite"
@ -10,8 +11,7 @@ build:
system_setup:
>
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz"
scripts:
build: "taskcluster/host-build.sh tflite"

6
taskcluster/linux-arm64-cpu-opt.yml
View File

@ -3,11 +3,11 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
- "index.project.deepspeech.deepspeech.native_client.arm64.${event.head.sha}"
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm64/artifacts/public/home.tar.xz"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>
@ -15,11 +15,11 @@ build:
wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb &&
echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb &&
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
system_config:
>
multistrap -d /tmp/multistrap-armbian64-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_armbian64_buster.conf
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm64/artifacts/public/home.tar.xz"
scripts:
build: "taskcluster/arm64-build.sh"
package: "taskcluster/package.sh"

7
taskcluster/linux-opt-base.tyml
View File

@ -46,11 +46,10 @@ then:
apt-get -qq -y remove --purge ubuntu-advantage-tools &&
${aptEc2Mirrors} &&
adduser --system --home ${system.homedir.linux} ${system.username} &&
apt-get -qq update && apt-get -qq -y install git pixz wget pkg-config libsox-dev && ${extraSystemSetup} &&
apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_trusty.apt} pixz pkg-config realpath unzip wget zip && ${extraSystemSetup} &&
cd ${system.homedir.linux}/ &&
echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
${system.homedir.linux}/DeepSpeech/tf/tc-apt.sh && ${extraSystemConfig} &&
echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package}

6
taskcluster/linux-rpi3-cpu-opt.yml
View File

@ -3,11 +3,11 @@ build:
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
- "index.project.deepspeech.deepspeech.native_client.arm.${event.head.sha}"
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm/artifacts/public/home.tar.xz"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>
@ -15,11 +15,11 @@ build:
wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb &&
echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb &&
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
system_config:
>
multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm/artifacts/public/home.tar.xz"
scripts:
build: "taskcluster/rpi3-build.sh"
package: "taskcluster/package.sh"

6
taskcluster/node-package-cpu.yml
View File

@ -1,5 +1,5 @@
build:
template_file: linux-opt-base.tyml
template_file: node-package-opt-base.tyml
dependencies:
- "darwin-amd64-cpu-opt"
- "linux-amd64-cpu-opt"
@ -9,9 +9,7 @@ build:
system_setup:
>
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz"
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
scripts:
build: "taskcluster/node-build.sh"
package: "taskcluster/node-package.sh"

6
taskcluster/node-package-gpu.yml
View File

@ -1,14 +1,12 @@
build:
template_file: linux-opt-base.tyml
template_file: node-package-opt-base.tyml
dependencies:
- "linux-amd64-gpu-opt"
- "win-amd64-gpu-opt"
system_setup:
>
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz"
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
scripts:
build: "taskcluster/node-build.sh --cuda"
package: "taskcluster/node-package.sh"

66
taskcluster/node-package-opt-base.tyml Normal file
View File

@ -0,0 +1,66 @@
$if: 'event.event in build.allowed'
then:
taskId: ${taskcluster.taskId}
provisionerId: ${taskcluster.docker.provisionerId}
workerType: ${taskcluster.docker.workerType}
taskGroupId: ${taskcluster.taskGroupId}
schedulerId: ${taskcluster.schedulerId}
dependencies:
$map: { $eval: build.dependencies }
each(b):
$eval: as_slugid(b)
created: { $fromNow: '0 sec' }
deadline: { $fromNow: '1 day' }
expires:
$if: '(event.event == "push") || (event.event == "tag")'
then: { $fromNow: '6 months' }
else: { $fromNow: '7 days' }
extra:
nc_asset_name: { $eval: build.nc_asset_name }
github:
$if: '(event.event == "push") || (event.event == "tag")'
then: { $eval: taskcluster.github_events.merge }
else: { $eval: taskcluster.github_events.pull_request }
routes:
$if: '(event.event == "push") || (event.event == "tag")'
then:
{ $eval: build.routes }
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
image: "ubuntu:14.04"
command:
- "/bin/bash"
- "--login"
- "-cxe"
- $let:
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
extraSystemConfig: { $eval: strip(str(build.system_config)) }
in: >
apt-get -qq -y remove --purge ubuntu-advantage-tools &&
${aptEc2Mirrors} &&
adduser --system --home ${system.homedir.linux} ${system.username} &&
apt-get -qq update && apt-get -qq -y install realpath git wget curl make && ${extraSystemSetup} &&
cd ${system.homedir.linux}/ &&
echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ~/DeepSpeech/tf/ && touch ~/DeepSpeech/tf/tc-vars.sh && chmod +x ~/DeepSpeech/tf/tc-vars.sh && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package}
artifacts:
"public":
type: "directory"
path: "/tmp/artifacts/"
expires:
$if: '(event.event == "push") || (event.event == "tag")'
then: { $fromNow: '6 months' }
else: { $fromNow: '7 days' }
metadata:
name: ${build.metadata.name}
description: ${build.metadata.description}
owner: ${event.head.user.email}
source: ${event.head.repo.url}

6
taskcluster/node-package-tflite.yml
View File

@ -1,5 +1,5 @@
build:
template_file: linux-opt-base.tyml
template_file: node-package-opt-base.tyml
dependencies:
- "darwin-amd64-tflite-opt"
- "linux-amd64-tflite-opt"
@ -7,9 +7,7 @@ build:
system_setup:
>
${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
apt-get -qq -y install ${python.packages_trusty.apt}
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz"
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
scripts:
build: "taskcluster/node-build.sh --tflite"
package: "taskcluster/node-package.sh"

37
taskcluster/pyenv-build.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PATH="${PYENV_ROOT}/bin:$PATH"
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
for pyver_conf in ${SUPPORTED_PYTHON_VERSIONS}; do
pyver=$(echo "${pyver_conf}" | cut -d':' -f1)
pyconf=$(echo "${pyver_conf}" | cut -d':' -f2)
pyalias="${pyver}_${pyconf}"
maybe_ssl102_py37 ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \
pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyalias}" "deepspeech"
virtualenv_activate "${pyalias}" "deepspeech"
python --version
python3 --version || true # Might fail without any issue on Windows
which pip
which pip3 || true # Might fail without any issue on Windows
virtualenv_deactivate "${pyalias}" "deepspeech"
done;

8
taskcluster/pyenv-darwin-amd64.yml Normal file
View File

@ -0,0 +1,8 @@
build:
template_file: pyenv-darwin-opt-base.tyml
scripts:
build: "taskcluster/pyenv-build.sh"
package: "taskcluster/pyenv-package.sh"
metadata:
name: "Builds Pyenv macOS AMD64"
description: "Setup a builds Pyenv for macOS/AMD64"

55
taskcluster/pyenv-darwin-opt-base.tyml Normal file
View File

@ -0,0 +1,55 @@
taskId: ${taskcluster.taskId}
provisionerId: ${taskcluster.generic.provisionerId}
workerType: ${taskcluster.generic.workerType}
taskGroupId: ${taskcluster.taskGroupId}
schedulerId: ${taskcluster.schedulerId}
dependencies:
$map: { $eval: build.dependencies }
each(b):
$eval: as_slugid(b)
created: { $fromNow: '0 sec' }
deadline: { $fromNow: '1 day' }
expires: { $fromNow: '6 months' }
scopes:
- "index:insert-task:project.deepspeech.*"
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
features:
taskclusterProxy: true
command:
- - "/bin/bash"
- "--login"
- "-cxe"
- $let:
taskIndexExpire: { $fromNow: '6 months' }
in: >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export LC_ALL=C &&
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
env &&
mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.osx.url}` &&
if [ "$swig_bin" != "200" ]; then
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} 127.0.0.1:8080 ${system.pyenv.osx.namespace}
fi;
artifacts:
- type: "directory"
path: "public/"
expires: { $fromNow: '6 months' }
metadata:
name: ${build.metadata.name}
description: ${build.metadata.description}
owner: ${event.head.user.email}
source: ${event.head.repo.url}

11
taskcluster/pyenv-linux-amd64.yml Normal file
View File

@ -0,0 +1,11 @@
build:
template_file: pyenv-linux-opt-base.tyml
system_setup:
>
apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_trusty.apt} wget
scripts:
build: "taskcluster/pyenv-build.sh"
package: "taskcluster/pyenv-package.sh"
metadata:
name: "Builds Pyenv Linux AMD64"
description: "Setup a builds Pyenv for Linux/AMD64"

52
taskcluster/pyenv-linux-opt-base.tyml Normal file
View File

@ -0,0 +1,52 @@
$if: 'event.event in build.allowed'
then:
taskId: ${taskcluster.taskId}
provisionerId: ${taskcluster.docker.provisionerId}
workerType: ${taskcluster.docker.workerType}
taskGroupId: ${taskcluster.taskGroupId}
schedulerId: ${taskcluster.schedulerId}
created: { $fromNow: '0 sec' }
deadline: { $fromNow: '1 day' }
expires: { $fromNow: '6 months' }
scopes:
- "index:insert-task:project.deepspeech.*"
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
image: ${build.docker_image}
features:
taskclusterProxy: true
command:
- "/bin/bash"
- "--login"
- "-cxe"
- $let:
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
taskIndexExpire: { $fromNow: '6 months' }
in: >
(apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
apt-get -qq update && apt-get -qq -y install curl git;
swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.linux.url}` &&
if [ "$swig_bin" != "200" ]; then
${extraSystemSetup} &&
adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ &&
mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts &&
echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.linux.namespace}
fi;
artifacts:
"public":
type: "directory"
path: "/tmp/artifacts/"
expires: { $fromNow: '6 months' }
metadata:
name: ${build.metadata.name}
description: ${build.metadata.description}
owner: ${event.head.user.email}
source: ${event.head.repo.url}

18
taskcluster/pyenv-package.sh Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash
set -xe
TC_EXPIRE=$1
TC_INSTANCE=$2
TC_INDEX=$3
source $(dirname "$0")/tc-tests-utils.sh
cd ${PYENV_ROOT}/ && $TAR -czf $TASKCLUSTER_ARTIFACTS/pyenv.tar.gz .
if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then
curl -sSL --fail -X PUT \
-H "Content-Type: application/json" \
-d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \
"http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}"
fi;

8
taskcluster/pyenv-win-amd64.yml Normal file
View File

@ -0,0 +1,8 @@
build:
template_file: pyenv-win-opt-base.tyml
scripts:
build: "taskcluster/pyenv-build.sh"
package: "taskcluster/pyenv-package.sh"
metadata:
name: "Builds Pyenv Windows AMD64"
description: "Setup a builds Pyenv for Windows/AMD64"

58
taskcluster/pyenv-win-opt-base.tyml Normal file
View File

@ -0,0 +1,58 @@
$if: 'event.event in build.allowed'
then:
taskId: ${taskcluster.taskId}
provisionerId: ${taskcluster.docker.provisionerId}
workerType: ${taskcluster.docker.workerTypeWin}
taskGroupId: ${taskcluster.taskGroupId}
schedulerId: ${taskcluster.schedulerId}
created: { $fromNow: '0 sec' }
deadline: { $fromNow: '1 day' }
expires: { $fromNow: '6 months' }
scopes:
- "index:insert-task:project.deepspeech.*"
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
features:
taskclusterProxy: true
mounts:
- file: msys2-base-x86_64.tar.xz
content:
sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c
url: >-
https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download
env:
TC_MSYS_VERSION: 'MSYS_NT-6.3'
MSYS: 'winsymlinks:nativestrict'
command:
- >-
"C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
"C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
- .\msys64\usr\bin\bash.exe --login -cx "exit"
- .\msys64\usr\bin\bash.exe --login -cx "pacman --noconfirm -Syu"
- $let:
taskIndexExpire: { $fromNow: '6 months' }
in: >
echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C &&
export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" &&
export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" &&
export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh &&
env && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${system.pyenv.win.url}`\" != \"200\" ]; then git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.win.namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k
- .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE"
artifacts:
- type: "directory"
path: "public/"
expires: { $fromNow: '6 months' }
metadata:
name: ${build.metadata.name}
description: ${build.metadata.description}
owner: ${event.head.user.email}
source: ${event.head.repo.url}

7
taskcluster/tc-all-vars.sh
View File

@ -5,12 +5,14 @@ set -xe
export OS=$(uname)
if [ "${OS}" = "Linux" ]; then
export DS_ROOT_TASK=${HOME}
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root"
export SWIG_ROOT="${HOME}/ds-swig"
export DS_CPU_COUNT=$(nproc)
fi;
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
export PYENV_ROOT="${TASKCLUSTER_TASK_DIR}/pyenv-root"
export SWIG_ROOT="$(cygpath ${USERPROFILE})/ds-swig"
export PLATFORM_EXE_SUFFIX=.exe
export DS_CPU_COUNT=$(nproc)
@ -20,6 +22,7 @@ if [ "${OS}" = "Darwin" ]; then
export SWIG_ROOT="${TASKCLUSTER_ORIG_TASKDIR}/ds-swig"
export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
export DS_CPU_COUNT=$(sysctl hw.ncpu |cut -d' ' -f2)
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root"
export HOMEBREW_NO_AUTO_UPDATE=1
export BREW_URL=https://github.com/Homebrew/brew/tarball/2.1.14
@ -52,6 +55,10 @@ if [ -f "${SWIG_ROOT}/bin/${DS_SWIG_BIN}" ]; then
swig -swiglib
fi;
PY37_OPENSSL_DIR="${PYENV_ROOT}/ssl-xenial"
export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/"
export LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH
export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts}
export TASKCLUSTER_TMP_DIR=${TASKCLUSTER_TMP_DIR:-/tmp}

60
taskcluster/tc-build-utils.sh
View File

@ -11,18 +11,8 @@ do_deepspeech_python_build()
unset PYTHON_BIN_PATH
unset PYTHONPATH
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/DeepSpeech/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
fi;
export PATH_WITHOUT_PYENV=${PATH}
export PATH="${PYENV_ROOT}/bin:$PATH"
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
mkdir -p wheels
SETUP_FLAGS=""
@ -38,26 +28,14 @@ do_deepspeech_python_build()
pyalias="${pyver}_${pyconf}"
export NUMPY_BUILD_VERSION="==1.7.0"
export NUMPY_DEP_VERSION=">=1.7.0"
maybe_numpy_min_version ${pyver}
maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \
pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyalias}" "deepspeech"
virtualenv_activate "${pyalias}" "deepspeech"
python --version
which pip
which pip3
# Set LD path because python ssl might require it
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \
EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \
EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \
@ -72,15 +50,8 @@ do_deepspeech_python_build()
make -C native_client/python/ bindings-clean
unset NUMPY_BUILD_VERSION
unset NUMPY_DEP_VERSION
virtualenv_deactivate "${pyalias}" "deepspeech"
done;
# If not, and if virtualenv_deactivate does not call "pyenv uninstall ${version}"
# we get stale python2 in PATH that blocks NodeJS builds
export PATH=${PATH_WITHOUT_PYENV}
}
do_deepspeech_decoder_build()
@ -90,43 +61,23 @@ do_deepspeech_decoder_build()
unset PYTHON_BIN_PATH
unset PYTHONPATH
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/DeepSpeech/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
mkdir -p wheels
for pyver_conf in ${SUPPORTED_PYTHON_VERSIONS}; do
pyver=$(echo "${pyver_conf}" | cut -d':' -f1)
pyconf=$(echo "${pyver_conf}" | cut -d':' -f2)
pyalias="${pyver}_${pyconf}"
export NUMPY_BUILD_VERSION="==1.7.0"
export NUMPY_DEP_VERSION=">=1.7.0"
maybe_numpy_min_version ${pyver}
maybe_ssl102_py37 ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \
pyenv_install ${pyver} "${pyalias}"
setup_pyenv_virtualenv "${pyalias}" "deepspeech"
virtualenv_activate "${pyalias}" "deepspeech"
python --version
which pip
which pip3
# Set LD path because python ssl might require it
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \
EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" \
EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \
@ -141,15 +92,8 @@ do_deepspeech_decoder_build()
make -C native_client/ctcdecode clean-keep-third-party
unset NUMPY_BUILD_VERSION
unset NUMPY_DEP_VERSION
virtualenv_deactivate "${pyalias}" "deepspeech"
done;
# If not, and if virtualenv_deactivate does not call "pyenv uninstall ${version}"
# we get stale python2 in PATH that blocks NodeJS builds
export PATH=${PATH_WITHOUT_PYENV}
}
do_deepspeech_nodejs_build()

30
taskcluster/tc-evaluate_tflite.sh
View File

@ -9,37 +9,13 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya
bitrate=$2
set_ldc_sample_filename "${bitrate}"
unset PYTHON_BIN_PATH
unset PYTHONPATH
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
download_data
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
PYENV_NAME=deepspeech-test
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "deepspeech"
deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type})
set -o pipefail
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat
pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat
pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt | cat
set +o pipefail
@ -52,4 +28,4 @@ pushd ${HOME}/DeepSpeech/ds/
python evaluate_tflite.py --model "${TASKCLUSTER_TMP_DIR}/${model_name_mmap}" --scorer data/smoke_test/pruned_lm.scorer --csv data/smoke_test/ldc93s1.csv
popd
virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"
virtualenv_deactivate "${pyalias}" "deepspeech"

126
taskcluster/tc-py-utils.sh
View File

@ -61,6 +61,34 @@ install_pyenv_virtualenv()
eval "$(pyenv virtualenv-init -)"
}
maybe_setup_virtualenv_cross_arm()
{
local version=$1
local name=$2
if [ -z "${PYENV_ROOT}" ]; then
echo "No PYENV_ROOT set";
exit 1;
fi;
if [ "${OS}" != "Linux" ]; then
echo "Only for Linux";
exit 1;
fi;
ARCH=$(uname -m)
if [ "${ARCH}" = "x86_64" ]; then
echo "Not for Linux/AMD64";
exit 0;
fi;
mkdir -p ${PYENV_ROOT}/versions/${version}/envs/
PIP_EXTRA_INDEX_URL="" python3 -m virtualenv -p python3 ${PYENV_ROOT}/versions/${version}/envs/${name}/
source ${PYENV_ROOT}/versions/${version}/envs/${name}/bin/activate
}
setup_pyenv_virtualenv()
{
local version=$1
@ -119,12 +147,6 @@ virtualenv_deactivate()
fi;
deactivate
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
rm -fr ${PYENV_ROOT}/versions/${version}/
else
pyenv uninstall --force ${name}
fi
}
pyenv_install()
@ -179,16 +201,11 @@ maybe_ssl102_py37()
pyver=$1
unset PY37_OPENSSL
unset PY37_LDPATH
unset PY37_SOURCE_PACKAGE
ARCH=$(uname -m)
case "${pyver}" in
3.7*|3.8*)
if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then
PY37_OPENSSL_DIR=${DS_ROOT_TASK}/ssl-xenial
if [ -d "${PY37_OPENSSL_DIR}" ]; then
rm -rf "${PY37_OPENSSL_DIR}"
fi
@ -210,9 +227,24 @@ maybe_ssl102_py37()
ln -sfn libssl.so.1.0.0 ${PY37_OPENSSL_DIR}/usr/lib/libssl.so
export PY37_OPENSSL="--with-openssl=${PY37_OPENSSL_DIR}/usr"
export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/"
fi;
;;
esac
}
maybe_numpy_min_version()
{
local pyver=$1
unset NUMPY_BUILD_VERSION
unset NUMPY_DEP_VERSION
# We set >= and < to make sure we have no numpy incompatibilities
# otherwise, `from deepspeech.impl` throws with "illegal instruction"
ARCH=$(uname -m)
case "${OS}:${ARCH}" in
Linux:x86_64)
case "${pyver}" in
3.7*)
export NUMPY_BUILD_VERSION="==1.14.5"
@ -224,35 +256,48 @@ maybe_ssl102_py37()
;;
esac
;;
esac
}
maybe_numpy_min_version_winamd64()
{
local pyver=$1
Darwin:*)
case "${pyver}" in
3.5*|3.6)
export NUMPY_BUILD_VERSION="==1.9.0"
export NUMPY_DEP_VERSION=">=1.9.0"
;;
3.7*)
export NUMPY_BUILD_VERSION="==1.14.5"
export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0"
;;
3.8*)
export NUMPY_BUILD_VERSION="==1.17.3"
export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3"
;;
esac
;;
if [ "${OS}" != "${TC_MSYS_VERSION}" ]; then
return;
fi
${TC_MSYS_VERSION}:x86_64)
case "${pyver}" in
3.5*)
export NUMPY_BUILD_VERSION="==1.11.0"
export NUMPY_DEP_VERSION=">=1.11.0,<1.12.0"
;;
3.6*)
export NUMPY_BUILD_VERSION="==1.12.0"
export NUMPY_DEP_VERSION=">=1.12.0,<1.14.5"
;;
3.7*)
export NUMPY_BUILD_VERSION="==1.14.5"
export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0"
;;
3.8*)
export NUMPY_BUILD_VERSION="==1.17.3"
export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3"
;;
esac
;;
# We set >= and < to make sure we have no numpy incompatibilities
# otherwise, `from deepspeech.impl` throws with "illegal instruction"
case "${pyver}" in
3.5*)
export NUMPY_BUILD_VERSION="==1.11.0"
export NUMPY_DEP_VERSION=">=1.11.0,<1.12.0"
;;
3.6*)
export NUMPY_BUILD_VERSION="==1.12.0"
export NUMPY_DEP_VERSION=">=1.12.0,<1.14.5"
;;
3.7*)
export NUMPY_BUILD_VERSION="==1.14.5"
export NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0"
;;
3.8*)
export NUMPY_BUILD_VERSION="==1.17.3"
export NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3"
*)
export NUMPY_BUILD_VERSION="==1.7.0"
export NUMPY_DEP_VERSION=">=1.7.0"
;;
esac
}
@ -294,8 +339,9 @@ extract_python_versions()
# 3.8.x => 38
local _pyver_pkg=$(echo "${_pyver}" | cut -d'.' -f1,2 | tr -d '.')
# UCS2 => narrow unicode
# UCS4 => wide unicode
# https://www.python.org/dev/peps/pep-3149/#proposal
# 'm' => pymalloc
# 'u' => wide unicode
local _py_unicode_type=$(echo "${_pyver_full}" | cut -d':' -f2)
if [ "${_py_unicode_type}" = "m" ]; then
local _pyconf="ucs2"

30
taskcluster/tc-python-tests-prod.sh
View File

@ -9,19 +9,6 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya
bitrate=$2
set_ldc_sample_filename "${bitrate}"
unset PYTHON_BIN_PATH
unset PYTHONPATH
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
model_source=${DEEPSPEECH_PROD_MODEL}
model_name=$(basename "${model_source}")
@ -30,24 +17,13 @@ model_name_mmap=$(basename "${model_source_mmap}")
download_data
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
PYENV_NAME=deepspeech-test
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "deepspeech"
deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type})
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat
run_prod_inference_tests "${bitrate}"
run_prod_concurrent_stream_tests "${bitrate}"
virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"
virtualenv_deactivate "${pyalias}" "deepspeech"

30
taskcluster/tc-python-tests.sh
View File

@ -9,40 +9,16 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya
bitrate=$2
set_ldc_sample_filename "${bitrate}"
unset PYTHON_BIN_PATH
unset PYTHONPATH
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
download_data
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
PYENV_NAME=deepspeech-test
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "deepspeech"
deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type})
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat
which deepspeech
deepspeech --version
run_all_inference_tests
virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"
virtualenv_deactivate "${pyalias}" "deepspeech"

30
taskcluster/tc-python_tflite-tests-prod.sh
View File

@ -9,19 +9,6 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya
bitrate=$2
set_ldc_sample_filename "${bitrate}"
unset PYTHON_BIN_PATH
unset PYTHONPATH
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
model_source=${DEEPSPEECH_PROD_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
@ -30,25 +17,16 @@ export DEEPSPEECH_ARTIFACTS_ROOT=${DEEPSPEECH_ARTIFACTS_TFLITE_ROOT}
download_data
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
maybe_setup_virtualenv_cross_arm "${pyalias}" "deepspeech"
maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
PYENV_NAME=deepspeech-test
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "deepspeech"
deepspeech_pkg_url=$(get_python_pkg_url "${pyver_pkg}" "${py_unicode_type}" "deepspeech_tflite")
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat
which deepspeech
deepspeech --version
run_prodtflite_inference_tests "${bitrate}"
virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"
virtualenv_deactivate "${pyalias}" "deepspeech"

30
taskcluster/tc-python_tflite-tests.sh
View File

@ -9,44 +9,22 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya
bitrate=$2
set_ldc_sample_filename "${bitrate}"
unset PYTHON_BIN_PATH
unset PYTHONPATH
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
model_name=$(basename "${model_source}")
model_name_mmap=$(basename "${model_source}")
download_data
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
maybe_setup_virtualenv_cross_arm "${pyalias}" "deepspeech"
maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
PYENV_NAME=deepspeech-test
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "deepspeech"
deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type})
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${deepspeech_pkg_url} | cat
which deepspeech
deepspeech --version
run_all_inference_tests
virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"
virtualenv_deactivate "${pyalias}" "deepspeech"

19
taskcluster/tc-single-shot-inference.sh
View File

@ -9,24 +9,11 @@ extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pya
ds=$2
frozen=$2
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PYENV_ROOT="${HOME}/ds-train/.pyenv"
export PATH="${PYENV_ROOT}/bin:${HOME}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
mkdir -p /tmp/train || true
mkdir -p /tmp/train_tflite || true
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
PYENV_NAME=deepspeech-train
PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv_install ${pyver}
setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}"
virtualenv_activate "${pyver}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "deepspeech"
set -o pipefail
pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat
@ -34,10 +21,10 @@ pip install --upgrade -r ${HOME}/DeepSpeech/ds/requirements.txt | cat
set +o pipefail
decoder_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type} "ds_ctcdecoder" "${DECODER_ARTIFACTS_ROOT}")
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${decoder_pkg_url} | cat
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: --upgrade ${decoder_pkg_url} | cat
pushd ${HOME}/DeepSpeech/ds/
time ./bin/run-tc-ldc93s1_singleshotinference.sh
popd
virtualenv_deactivate "${pyver}" "${PYENV_NAME}"
virtualenv_deactivate "${pyver}" "deepspeech"

43
taskcluster/tc-train-tests.sh
View File

@ -4,43 +4,15 @@ set -xe
source $(dirname "$0")/tc-tests-utils.sh
pyver_full=$1
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
bitrate=$2
if [ -z "${pyver_full}" ]; then
echo "No python version given, aborting."
exit 1
fi;
pyver=$(echo "${pyver_full}" | cut -d':' -f1)
# 2.7.x => 27
pyver_pkg=$(echo "${pyver}" | cut -d'.' -f1,2 | tr -d '.')
py_unicode_type=$(echo "${pyver_full}" | cut -d':' -f2)
if [ "${py_unicode_type}" = "m" ]; then
pyconf="ucs2"
elif [ "${py_unicode_type}" = "mu" ]; then
pyconf="ucs4"
fi;
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PYENV_ROOT="${HOME}/ds-train/.pyenv"
export PATH="${PYENV_ROOT}/bin:${HOME}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
mkdir -p /tmp/train || true
mkdir -p /tmp/train_tflite || true
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
PYENV_NAME=deepspeech-train
PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver}
pyenv virtualenv ${pyver} ${PYENV_NAME}
source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate
virtualenv_activate "${pyalias}" "deepspeech"
set -o pipefail
pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat
@ -51,12 +23,7 @@ pushd ${HOME}/DeepSpeech/ds/
verify_ctcdecoder_url
popd
platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));')
whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')"
decoder_pkg="ds_ctcdecoder-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl"
decoder_pkg_url=${DECODER_ARTIFACTS_ROOT}/${decoder_pkg}
decoder_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type} "ds_ctcdecoder" "${DECODER_ARTIFACTS_ROOT}")
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} ${decoder_pkg_url} | cat
# Prepare correct arguments for training
@ -97,4 +64,4 @@ pushd ${HOME}/DeepSpeech/ds/
time ./bin/run-tc-ldc93s1_checkpoint.sh
popd
deactivate
virtualenv_deactivate "${pyalias}" "deepspeech"

42
taskcluster/tc-transfer-tests.sh
View File

@ -4,42 +4,13 @@ set -xe
source $(dirname "$0")/tc-tests-utils.sh
pyver_full=$1
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
if [ -z "${pyver_full}" ]; then
echo "No python version given, aborting."
exit 1
fi;
pyver=$(echo "${pyver_full}" | cut -d':' -f1)
# 2.7.x => 27
pyver_pkg=$(echo "${pyver}" | cut -d'.' -f1,2 | tr -d '.')
py_unicode_type=$(echo "${pyver_full}" | cut -d':' -f2)
if [ "${py_unicode_type}" = "m" ]; then
pyconf="ucs2"
elif [ "${py_unicode_type}" = "mu" ]; then
pyconf="ucs4"
fi;
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PYENV_ROOT="${HOME}/ds-train/.pyenv"
export PATH="${PYENV_ROOT}/bin:${HOME}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
mkdir -p /tmp/train || true
mkdir -p /tmp/train_tflite || true
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
PYENV_NAME=deepspeech-train
PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv install ${pyver}
pyenv virtualenv ${pyver} ${PYENV_NAME}
source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate
virtualenv_activate "${pyalias}" "deepspeech"
set -o pipefail
pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat
@ -50,16 +21,11 @@ pushd ${HOME}/DeepSpeech/ds/
verify_ctcdecoder_url
popd
platform=$(python -c 'import sys; import platform; plat = platform.system().lower(); arch = platform.machine().lower(); plat = "manylinux1" if plat == "linux" and arch == "x86_64" else plat; plat = "macosx_10_10" if plat == "darwin" else plat; sys.stdout.write("%s_%s" % (plat, platform.machine()));')
whl_ds_version="$(python -c 'from pkg_resources import parse_version; print(parse_version("'${DS_VERSION}'"))')"
decoder_pkg="ds_ctcdecoder-${whl_ds_version}-cp${pyver_pkg}-cp${pyver_pkg}${py_unicode_type}-${platform}.whl"
decoder_pkg_url=${DECODER_ARTIFACTS_ROOT}/${decoder_pkg}
decoder_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type} "ds_ctcdecoder" "${DECODER_ARTIFACTS_ROOT}")
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} ${decoder_pkg_url} | cat
pushd ${HOME}/DeepSpeech/ds/
time ./bin/run-tc-transfer.sh
popd
deactivate
virtualenv_deactivate "${pyalias}" "deepspeech"

15
taskcluster/test-darwin-opt-base.tyml
View File

@ -45,8 +45,8 @@ then:
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
in: >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" &&
export LC_ALL=C &&
@ -55,20 +55,21 @@ then:
export PIP_DEFAULT_TIMEOUT=60 &&
env &&
trap "export TASKCLUSTER_TASK_EXIT_CODE=$? &&
(mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ || true) &&
cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ &&
exit $TASKCLUSTER_TASK_EXIT_CODE" 0 &&
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
cd $TASKCLUSTER_TASK_DIR &&
(mkdir pyenv-root/ && gtar -C pyenv-root/ -xf $TASKCLUSTER_ORIG_TASKDIR/pyenv.tar.gz) &&
(mkdir homebrew-tests/ && gtar -C homebrew-tests/ -xf $TASKCLUSTER_ORIG_TASKDIR/homebrew-tests.tar.gz) &&
${extraSystemSetup} &&
/bin/bash ${build.args.tests_cmdline}
mounts:
- cacheName: deepspeech-macos-pyenv
directory: pyenv.cache/
- directory: ../tc-workdir/homebrew-tests
format: tar.gz
- file: pyenv.tar.gz
content:
url: ${system.pyenv.osx.url}
- file: homebrew-tests.tar.gz
content:
url: ${system.homebrew_tests.url}

5
taskcluster/test-evaluate_tflite-linux-amd64-py36m-opt.yml
View File

@ -4,11 +4,8 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-evaluate_tflite.sh 3.6.4:m 16k"
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-evaluate_tflite.sh 3.6.10:m 16k"
metadata:
name: "DeepSpeech Linux AMD64 CPU evaluate_tflite.py Py3.6 (16kHz)"
description: "Test evaluate_tflite.py on Linux/AMD64 using upstream TensorFlow Python 3.6, CPU only, optimized version"

2
taskcluster/test-linux-opt-base.tyml
View File

@ -48,7 +48,7 @@ then:
apt-get -qq update && apt-get -qq -y install curl python-simplejson git pixz sox sudo wget && ${extraSystemSetup} &&
adduser --system --home ${system.homedir.linux} ${system.username} &&
cd ${system.homedir.linux} &&
echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}&& mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
sudo -H -u ${system.username} --preserve-env /bin/bash ${build.args.tests_cmdline}

3
taskcluster/test-python_35_16k-linux-amd64-opt.yml
View File

@ -4,9 +4,6 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.8:m 16k"
metadata:

3
taskcluster/test-python_35_16k-linux-amd64-prod_pbmodel-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.5.8:m 16k"
metadata:

3
taskcluster/test-python_35_8k-linux-amd64-opt.yml
View File

@ -4,9 +4,6 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_8k-linux-amd64-py36m-opt"
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.8:m 8k"
metadata:

3
taskcluster/test-python_35_8k-linux-amd64-prod_pbmodel-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.5.8:m 8k"
metadata:

3
taskcluster/test-python_35_tflite_16k-linux-amd64-prod-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.5.8:m 16k"
metadata:

3
taskcluster/test-python_35_tflite_8k-linux-amd64-prod-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.5.8:m 8k"
metadata:

3
taskcluster/test-python_36_16k-linux-amd64-opt.yml
View File

@ -4,9 +4,6 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.6.10:m 16k"
metadata:

3
taskcluster/test-python_36_16k-linux-amd64-prod_pbmodel-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.6.10:m 16k"
metadata:

3
taskcluster/test-python_36_8k-linux-amd64-opt.yml
View File

@ -4,9 +4,6 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_8k-linux-amd64-py36m-opt"
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.6.10:m 8k"
metadata:

3
taskcluster/test-python_36_8k-linux-amd64-prod_pbmodel-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.6.10:m 8k"
metadata:

3
taskcluster/test-python_36_tflite_16k-linux-amd64-prod-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.6.10:m 16k"
metadata:

3
taskcluster/test-python_36_tflite_8k-linux-amd64-prod-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.6.10:m 8k"
metadata:

3
taskcluster/test-python_37_16k-linux-amd64-opt.yml
View File

@ -4,9 +4,6 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.6:m 16k"
metadata:

3
taskcluster/test-python_37_16k-linux-amd64-prod_pbmodel-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.7.6:m 16k"
metadata:

3
taskcluster/test-python_37_8k-linux-amd64-opt.yml
View File

@ -4,9 +4,6 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_8k-linux-amd64-py36m-opt"
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.6:m 8k"
metadata:

3
taskcluster/test-python_37_8k-linux-amd64-prod_pbmodel-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.7.6:m 8k"
metadata:

3
taskcluster/test-python_37_tflite_16k-linux-amd64-prod-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.7.6:m 16k"
metadata:

3
taskcluster/test-python_37_tflite_8k-linux-amd64-prod-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.7.6:m 8k"
metadata:

3
taskcluster/test-python_38_16k-linux-amd64-opt.yml
View File

@ -4,9 +4,6 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.1: 16k"
metadata:

3
taskcluster/test-python_38_16k-linux-amd64-prod_pbmodel-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.8.1: 16k"
metadata:

3
taskcluster/test-python_38_8k-linux-amd64-opt.yml
View File

@ -4,9 +4,6 @@ build:
- "linux-amd64-cpu-opt"
- "test-training_8k-linux-amd64-py36m-opt"
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.1: 8k"
metadata:

3
taskcluster/test-python_38_8k-linux-amd64-prod_pbmodel-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-cpu-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python-tests-prod.sh 3.8.1: 8k"
metadata:

3
taskcluster/test-python_38_tflite_16k-linux-amd64-prod-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.8.1: 16k"
metadata:

3
taskcluster/test-python_38_tflite_8k-linux-amd64-prod-opt.yml
View File

@ -2,9 +2,6 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-tflite-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-python_tflite-tests-prod.sh 3.8.1: 8k"
metadata:

5
taskcluster/test-singleshotinference-linux-amd64-py36m-opt.yml
View File

@ -2,11 +2,8 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-ctc-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-single-shot-inference.sh 3.6.4:m"
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-single-shot-inference.sh 3.6.10:m"
metadata:
name: "DeepSpeech Linux AMD64 CPU single-shot inference Py3.6"
description: "Single-shot inference a DeepSpeech LDC93S1 checkpoint for Linux/AMD64 using upstream TensorFlow Python 3.6, CPU only, optimized version"

5
taskcluster/test-training_16k-linux-amd64-py35m-opt.yml
View File

@ -2,11 +2,8 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-ctc-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.5.5:m 16k"
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.5.8:m 16k"
metadata:
name: "DeepSpeech Linux AMD64 CPU 16kHz training Py3.5"
description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 16kHz Python 3.5, CPU only, optimized version"

5
taskcluster/test-training_16k-linux-amd64-py36m-opt.yml
View File

@ -2,11 +2,8 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-ctc-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.4:m 16k"
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 16k"
metadata:
name: "DeepSpeech Linux AMD64 CPU 16kHz training Py3.6"
description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 16kHz Python 3.6, CPU only, optimized version"

5
taskcluster/test-training_8k-linux-amd64-py36m-opt.yml
View File

@ -2,11 +2,8 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-ctc-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.4:m 8k"
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 8k"
metadata:
name: "DeepSpeech Linux AMD64 CPU 8kHz training Py3.6"
description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 8kHz Python 3.6, CPU only, optimized version"

5
taskcluster/test-transfer-linux-amd64-py36m-opt.yml
View File

@ -2,11 +2,8 @@ build:
template_file: test-linux-opt-base.tyml
dependencies:
- "linux-amd64-ctc-opt"
system_setup:
>
apt-get -qq -y install ${python.packages_trusty.apt}
args:
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-transfer-tests.sh 3.6.4:m"
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-transfer-tests.sh 3.6.10:m"
metadata:
name: "DeepSpeech Linux AMD64 CPU transfer learning Py3.6"
description: "Training a DeepSpeech LDC93S1 model with transfer learning for Linux/AMD64 16kHz Python 3.6, CPU only, optimized version"

17
taskcluster/test-win-opt-base.tyml
View File

@ -19,12 +19,6 @@ then:
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
mounts:
- file: msys2-base-x86_64.tar.xz
content:
sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c
url: >-
https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download
env:
$let:
@ -63,6 +57,7 @@ then:
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
cd $TASKCLUSTER_TASK_DIR &&
(mkdir pyenv-root/ && 7z x -so $USERPROFILE/pyenv.tar.gz | 7z x -opyenv-root/ -aoa -ttar -si ) &&
pacman --noconfirm -R bsdtar &&
pacman --noconfirm -S tar vim &&
${extraSystemSetup} &&
@ -70,6 +65,16 @@ then:
export TASKCLUSTER_TASK_EXIT_CODE=$? &&
cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE"
mounts:
- file: msys2-base-x86_64.tar.xz
content:
sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c
url: >-
https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download
- file: pyenv.tar.gz
content:
url: ${system.pyenv.win.url}
metadata:
name: ${build.metadata.name}
description: ${build.metadata.description}

1
taskcluster/win-amd64-cpu-opt.yml
View File

@ -7,6 +7,7 @@ build:
dependencies:
- "node-gyp-cache"
- "swig-win-amd64"
- "pyenv-win-amd64"
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz"
scripts:
build: "taskcluster/win-build.sh"

1
taskcluster/win-amd64-gpu-opt.yml
View File

@ -7,6 +7,7 @@ build:
dependencies:
- "node-gyp-cache"
- "swig-win-amd64"
- "pyenv-win-amd64"
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win-cuda/artifacts/public/home.tar.xz"
scripts:
build: "taskcluster/win-build.sh --cuda"

1
taskcluster/win-amd64-tflite-opt.yml
View File

@ -7,6 +7,7 @@ build:
dependencies:
- "node-gyp-cache"
- "swig-win-amd64"
- "pyenv-win-amd64"
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz"
scripts:
build: "taskcluster/win-build.sh --tflite"

39
taskcluster/win-opt-base.tyml
View File

@ -28,25 +28,10 @@ routes:
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
mounts:
- file: msys2-base-x86_64.tar.xz
content:
sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c
url: >-
https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download
- directory: .node-gyp
format: tar.gz
content:
url: ${system.node_gyp_cache.url}
- directory: ds-swig
format: tar.gz
content:
url: ${system.swig.cache.win_amd64}
env:
TC_MSYS_VERSION: 'MSYS_NT-6.3'
MSYS: 'winsymlinks:nativestrict'
TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow}
EXAMPLES_CLONE_URL: "https://github.com/mozilla/DeepSpeech-examples"
EXAMPLES_CHECKOUT_TARGET: "master"
@ -64,13 +49,14 @@ payload:
(mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR &&
env &&
ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64 &&
(wget -O - $TENSORFLOW_BUILD_ARTIFACT | 7z x -txz -si -so | 7z x -aoa -ttar -si ) &&
(7z x -txz -so $USERPROFILE/home.tar.xz | 7z x -aoa -ttar -si ) &&
git clone --quiet $EXAMPLES_CLONE_URL $TASKCLUSTER_TASK_DIR/DeepSpeech/examples &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && git checkout --quiet $EXAMPLES_CHECKOUT_TARGET &&
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
cd $TASKCLUSTER_TASK_DIR &&
(mkdir pyenv-root/ && 7z x -so $USERPROFILE/pyenv.tar.gz | 7z x -opyenv-root/ -aoa -ttar -si ) &&
pacman --noconfirm -R bsdtar &&
pacman --noconfirm -S tar make &&
pacman --noconfirm -S zip &&
@ -83,6 +69,27 @@ payload:
- .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh &&
exit $TASKCLUSTER_TASK_EXIT_CODE"
mounts:
- file: msys2-base-x86_64.tar.xz
content:
sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c
url: >-
https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download
- file: home.tar.xz
content:
url: ${build.tensorflow}
- file: pyenv.tar.gz
content:
url: ${system.pyenv.win.url}
- directory: .node-gyp
format: tar.gz
content:
url: ${system.node_gyp_cache.url}
- directory: ds-swig
format: tar.gz
content:
url: ${system.swig.cache.win_amd64}
artifacts:
- type: "directory"
path: "public/"