experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2756 from lissyx/no-remirror

Browse Source 
Manually update the TC index when needed
This commit is contained in:
lissyx 2020-02-14 14:52:45 +01:00 committed by GitHub
commit 551bded23e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 72 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.taskcluster.yml
View File

@ -27,6 +27,7 @@ tasks:
scopes: [
"queue:create-task:highest:proj-deepspeech/*",
"queue:route:index.project.deepspeech.*",
"index:insert-task:project.deepspeech.*",
"queue:route:notify.irc-channel.*",
"queue:scheduler-id:taskcluster-github",
"generic-worker:cache:deepspeech-homebrew-bin",

12
taskcluster/.shared.yml
View File

@ -59,14 +59,16 @@ cpp:
packages: 'install_pkg_local_homebrew "sox"'
env: 'export EXTRA_ENV="DYLD_LIBRARY_PATH=$LOCAL_HOMEBREW_DIRECTORY/lib/:$DYLD_LIBRARY_PATH"'
system:
node_gyp_cache: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.0/artifacts/public/node-gyp-cache.tar.gz'
node_gyp_cache:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.1/artifacts/public/node-gyp-cache.tar.gz'
namespace: 'project.deepspeech.node-gyp-cache.1'
swig:
repo: "https://github.com/lissyx/swig"
sha1: "04baa423c41008712e811cd5c1ee94e9e90894aa"
sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
cache:
linux_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.04baa423c41008712e811cd5c1ee94e9e90894aa/artifacts/public/ds-swig.tar.gz'
darwin_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.04baa423c41008712e811cd5c1ee94e9e90894aa/artifacts/public/ds-swig.tar.gz'
win_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.04baa423c41008712e811cd5c1ee94e9e90894aa/artifacts/public/ds-swig.tar.gz'
linux_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
darwin_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
win_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
username: 'build-user'
homedir:
linux: '/home/build-user'

1
taskcluster/android-arm64-cpu-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64"

1
taskcluster/android-armv7-cpu-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7"

1
taskcluster/android-x86_64-cpu-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64"

1
taskcluster/darwin-amd64-cpu-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: darwin-opt-base.tyml
dependencies:
- "swig-darwin-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx"

1
taskcluster/darwin-amd64-ctc-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: darwin-opt-base.tyml
dependencies:
- "swig-darwin-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc"

1
taskcluster/darwin-amd64-tflite-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: darwin-opt-base.tyml
dependencies:
- "swig-darwin-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite"

4
taskcluster/darwin-opt-base.tyml
View File

@ -125,7 +125,7 @@ payload:
- directory: ${system.homedir.osx}/Library/Caches/node-gyp/
format: tar.gz
content:
url: ${system.node_gyp_cache}
url: ${system.node_gyp_cache.url}
- directory: ds-swig
format: tar.gz
content:
@ -134,7 +134,7 @@ payload:
- directory: ${system.homedir.osx}/Library/Caches/node-gyp/
format: tar.gz
content:
url: ${system.node_gyp_cache}
url: ${system.node_gyp_cache.url}
- directory: ds-swig
format: tar.gz
content:

1
taskcluster/linux-amd64-cpu-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu"

1
taskcluster/linux-amd64-ctc-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-ctc"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-ctc"

1
taskcluster/linux-amd64-gpu-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu"

1
taskcluster/linux-amd64-tflite-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite"

1
taskcluster/linux-arm64-cpu-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"

2
taskcluster/linux-opt-base.tyml
View File

@ -48,7 +48,7 @@ then:
adduser --system --home ${system.homedir.linux} ${system.username} &&
apt-get -qq update && apt-get -qq -y install git pixz wget pkg-config libsox-dev && ${extraSystemSetup} &&
cd ${system.homedir.linux}/ &&
echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
${system.homedir.linux}/DeepSpeech/tf/tc-apt.sh && ${extraSystemConfig} &&
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&

1
taskcluster/linux-rpi3-cpu-opt.yml
View File

@ -2,6 +2,7 @@ build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"

18
taskcluster/node-gyp-cache-base.tyml
View File

@ -8,12 +8,16 @@ then:
created: { $fromNow: '0 sec' }
deadline: { $fromNow: '1 day' }
expires: { $fromNow: '6 months' }
routes: { $eval: build.routes }
scopes:
- "index:insert-task:project.deepspeech.*"
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
image: "node:12"
features:
taskclusterProxy: true
# This task will inspect system.node_gyp_cache taskcluster index existence:
# - if the artifact does not exists, it will build it
# - if the artifact exists, it will re-mirror it (if we don't do that, new
@ -24,18 +28,20 @@ then:
- "-cxe"
- $let:
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
extraSystemConfig: { $eval: strip(str(build.system_config)) }
taskIndexExpire: { $fromNow: '6 months' }
in: >
apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache}` &&
cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache.url}` &&
if [ "$cache_file" != "200" ]; then
mkdir -p ~/DeepSpeech/ds/ &&
git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ &&
cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
~/DeepSpeech/ds/${build.scripts.build} &&
~/DeepSpeech/ds/${build.scripts.package}
else
mkdir -p /tmp/artifacts/ && curl -sSL -o /tmp/artifacts/node-gyp-cache.tar.gz ${system.node_gyp_cache}
~/DeepSpeech/ds/${build.scripts.package} &&
curl -sSL --fail -X PUT \
-H "Content-Type: application/json" \
-d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
"http://taskcluster/index/v1/task/${system.node_gyp_cache.namespace}"
fi;
artifacts:

2
taskcluster/node-gyp-cache.yml
View File

@ -1,7 +1,5 @@
build:
template_file: node-gyp-cache-base.tyml
routes:
- "index.project.deepspeech.node-gyp-cache.0"
scripts:
build: "taskcluster/node-gyp-populate.sh"
package: "taskcluster/node-gyp-package.sh"

45
taskcluster/swig-darwin-opt-base.tyml
View File

@ -10,8 +10,8 @@ dependencies:
created: { $fromNow: '0 sec' }
deadline: { $fromNow: '1 day' }
expires: { $fromNow: '6 months' }
routes:
- "index.project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
scopes:
- "index:insert-task:project.deepspeech.*"
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
@ -20,24 +20,29 @@ payload:
- - "/bin/bash"
- "--login"
- "-cxe"
- >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_TASKDIR="$(pwd)" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
export LC_ALL=C &&
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
export HOMEBREW_NO_AUTO_UPDATE=1 &&
env &&
swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` &&
if [ "$swig_bin" != "200" ]; then
git clone --quiet ${system.swig.repo} $TASKCLUSTER_TASKDIR/swig/ &&
cd $TASKCLUSTER_TASKDIR/swig/ && git checkout --quiet ${system.swig.sha1} &&
$TASKCLUSTER_TASKDIR/swig/${build.scripts.build} &&
$TASKCLUSTER_TASKDIR/swig/${build.scripts.package}
else
mkdir -p $TASKCLUSTER_ARTIFACTS/ && curl -sSL -o $TASKCLUSTER_ARTIFACTS/ds-swig.tar.gz https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz
fi;
- $let:
taskIndexExpire: { $fromNow: '6 months' }
in: >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_TASKDIR="$(pwd)" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
export LC_ALL=C &&
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
export HOMEBREW_NO_AUTO_UPDATE=1 &&
env &&
mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` &&
if [ "$swig_bin" != "200" ]; then
git clone --quiet ${system.swig.repo} $TASKCLUSTER_TASKDIR/swig/ &&
cd $TASKCLUSTER_TASKDIR/swig/ && git checkout --quiet ${system.swig.sha1} &&
$TASKCLUSTER_TASKDIR/swig/${build.scripts.build} &&
$TASKCLUSTER_TASKDIR/swig/${build.scripts.package} &&
curl -sSL --fail -X PUT \
-H "Content-Type: application/json" \
-d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
"http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
fi;
artifacts:
- type: "directory"

17
taskcluster/swig-linux-opt-base.tyml
View File

@ -8,20 +8,23 @@ then:
created: { $fromNow: '0 sec' }
deadline: { $fromNow: '1 day' }
expires: { $fromNow: '6 months' }
routes:
- "index.project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
scopes:
- "index:insert-task:project.deepspeech.*"
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
image: ${build.docker_image}
features:
taskclusterProxy: true
command:
- "/bin/bash"
- "--login"
- "-cxe"
- $let:
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
extraSystemConfig: { $eval: strip(str(build.system_config)) }
taskIndexExpire: { $fromNow: '6 months' }
in: >
(apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
@ -31,9 +34,11 @@ then:
echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${system.swig.repo} ~/swig/ && cd ~/swig/ && git checkout --quiet ${system.swig.sha1})" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/swig/${build.scripts.build} &&
sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/swig/${build.scripts.package}
else
mkdir -p /tmp/artifacts/ && curl -sSL -o /tmp/artifacts/ds-swig.tar.gz https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz
sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/swig/${build.scripts.package} &&
curl -sSL --fail -X PUT \
-H "Content-Type: application/json" \
-d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
"http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
fi;
artifacts:

2
taskcluster/win-opt-base.tyml
View File

@ -37,7 +37,7 @@ payload:
- directory: .node-gyp
format: tar.gz
content:
url: ${system.node_gyp_cache}
url: ${system.node_gyp_cache.url}
- directory: ds-swig
format: tar.gz
content: