diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 00fc663c..502548dd 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -80,6 +80,7 @@ cpp: packages: 'install_pkg_local_homebrew "sox"' env: 'export EXTRA_ENV="DYLD_LIBRARY_PATH=$LOCAL_HOMEBREW_DIRECTORY/lib/:$DYLD_LIBRARY_PATH"' system: + node_gyp_cache: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.0/artifacts/public/node-gyp-cache.tar.gz' username: 'build-user' homedir: linux: '/home/build-user' diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index a70d6199..594b6b64 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -112,6 +112,8 @@ payload: then: { $fromNow: '6 months' } else: { $fromNow: '7 days' } + # Define node gyp cache mount twice because we cannot have it separated + # from the if/then without the parser to break mounts: $if: '(event.event != "push") && (event.event != "tag")' then: @@ -121,6 +123,15 @@ payload: directory: homebrew.cache/ - cacheName: deepspeech-macos-pyenv directory: pyenv.cache/ + - directory: ${system.homedir.osx}/Library/Caches/node-gyp/ + format: tar.gz + content: + url: ${system.node_gyp_cache} + else: + - directory: ${system.homedir.osx}/Library/Caches/node-gyp/ + format: tar.gz + content: + url: ${system.node_gyp_cache} metadata: name: ${build.metadata.name} diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index 4f7c5fa3..41e1971f 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -48,7 +48,7 @@ then: apt-get -qq update && apt-get -qq -y install git pixz wget pkg-config libsox-dev && ${extraSystemSetup} && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${system.homedir.linux}/DeepSpeech/tf/tc-apt.sh && ${extraSystemConfig} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && diff --git a/taskcluster/node-gyp-cache-base.tyml b/taskcluster/node-gyp-cache-base.tyml new file mode 100644 index 00000000..e4c3d05c --- /dev/null +++ b/taskcluster/node-gyp-cache-base.tyml @@ -0,0 +1,51 @@ +$if: 'event.event in build.allowed' +then: + taskId: ${taskcluster.taskId} + provisionerId: ${taskcluster.docker.provisionerId} + workerType: ${taskcluster.docker.workerType} + taskGroupId: ${taskcluster.taskGroupId} + schedulerId: ${taskcluster.schedulerId} + created: { $fromNow: '0 sec' } + deadline: { $fromNow: '1 day' } + expires: { $fromNow: '6 months' } + routes: { $eval: build.routes } + + payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + image: "node:12" + + # This task will inspect system.node_gyp_cache taskcluster index existence: + # - if the artifact does not exists, it will build it + # - if the artifact exists, it will re-mirror it (if we don't do that, new + # index gets published with no artifact and erases existing one) + command: + - "/bin/bash" + - "--login" + - "-cxe" + - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + extraSystemConfig: { $eval: strip(str(build.system_config)) } + in: > + apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; + cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache}` && + if [ "$cache_file" != "200" ]; then + mkdir -p ~/DeepSpeech/ds/ && + git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && + cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && + ~/DeepSpeech/ds/${build.scripts.build} && + ~/DeepSpeech/ds/${build.scripts.package} + else + mkdir -p /tmp/artifacts/ && curl -sSL -o /tmp/artifacts/node-gyp-cache.tar.gz ${system.node_gyp_cache} + fi; + + artifacts: + "public": + type: "directory" + path: "/tmp/artifacts/" + expires: { $fromNow: '6 months' } + + metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/node-gyp-cache.yml b/taskcluster/node-gyp-cache.yml new file mode 100644 index 00000000..fd314704 --- /dev/null +++ b/taskcluster/node-gyp-cache.yml @@ -0,0 +1,10 @@ +build: + template_file: node-gyp-cache-base.tyml + routes: + - "index.project.deepspeech.node-gyp-cache.0" + scripts: + build: "taskcluster/node-gyp-populate.sh" + package: "taskcluster/node-gyp-package.sh" + metadata: + name: "Cache for node-gyp headers" + description: "Building cache for node-gyp headers" diff --git a/taskcluster/node-gyp-package.sh b/taskcluster/node-gyp-package.sh new file mode 100755 index 00000000..b17faf32 --- /dev/null +++ b/taskcluster/node-gyp-package.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +cd $DS_ROOT_TASK/node-gyp-cache/ && tar -czf ${TASKCLUSTER_ARTIFACTS}/node-gyp-cache.tar.gz . diff --git a/taskcluster/node-gyp-populate.sh b/taskcluster/node-gyp-populate.sh new file mode 100755 index 00000000..ebc0e47a --- /dev/null +++ b/taskcluster/node-gyp-populate.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +node --version + +npm --version + +npm install -g node-gyp@6.x + +devDir=$DS_ROOT_TASK/node-gyp-cache/ + +mkdir -p ${devDir} + +node-gyp list ${devDir} + +for node in ${SUPPORTED_NODEJS_VERSIONS}; do + node-gyp install --devdir ${devDir} \ + --target=$node + mkdir ${devDir}/${node}/x64/ || true + curl -sSL https://nodejs.org/dist/v${node}/win-x64/node.lib -o ${devDir}/${node}/x64/node.lib +done; + +for electron in ${SUPPORTED_ELECTRONJS_VERSIONS}; do + node-gyp install --devdir ${devDir} \ + --target=$electron \ + --disturl=https://electronjs.org/headers \ + --runtime=electron + mkdir ${devDir}/${electron}/x64/ || true + curl -sSL https://electronjs.org/headers/v${electron}/win-x64/node.lib -o ${devDir}/${electron}/x64/node.lib +done; diff --git a/taskcluster/tc-tests-utils.sh b/taskcluster/tc-tests-utils.sh index f8bf07c1..4841afaf 100755 --- a/taskcluster/tc-tests-utils.sh +++ b/taskcluster/tc-tests-utils.sh @@ -59,6 +59,10 @@ model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}" ldc93s1_sample_filename='' SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-3.5.8:ucs4 3.6.10:ucs4 3.7.6:ucs4 3.8.1:ucs4} + +# When updating NodeJS / ElectronJS supported versions, do not forget to increment +# deepspeech.node-gyp-cache. in both `system.node_gyp_cache` (taskcluster/.shared.yml) +# and route index (taskcluster/node-gyp-cache.yml) to ensure the cache is updated SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-10.18.1 11.15.0 12.8.1 13.1.0} SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-5.0.13 6.0.12 6.1.7 7.0.1 7.1.8} diff --git a/taskcluster/win-amd64-cpu-opt.yml b/taskcluster/win-amd64-cpu-opt.yml index 0826cead..5083cdfc 100644 --- a/taskcluster/win-amd64-cpu-opt.yml +++ b/taskcluster/win-amd64-cpu-opt.yml @@ -6,6 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.win.${event.head.sha}" - "notify.irc-channel.${notifications.irc}.on-exception" - "notify.irc-channel.${notifications.irc}.on-failed" + dependencies: + - "node-gyp-cache" tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh" diff --git a/taskcluster/win-amd64-gpu-opt.yml b/taskcluster/win-amd64-gpu-opt.yml index db8b0f3f..7388e16a 100644 --- a/taskcluster/win-amd64-gpu-opt.yml +++ b/taskcluster/win-amd64-gpu-opt.yml @@ -6,6 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.win-cuda.${event.head.sha}" - "notify.irc-channel.${notifications.irc}.on-exception" - "notify.irc-channel.${notifications.irc}.on-failed" + dependencies: + - "node-gyp-cache" tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win-cuda/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh --cuda" diff --git a/taskcluster/win-amd64-tflite-opt.yml b/taskcluster/win-amd64-tflite-opt.yml index feb302e4..405deb58 100644 --- a/taskcluster/win-amd64-tflite-opt.yml +++ b/taskcluster/win-amd64-tflite-opt.yml @@ -6,6 +6,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.win-tflite.${event.head.sha}" - "notify.irc-channel.${notifications.irc}.on-exception" - "notify.irc-channel.${notifications.irc}.on-failed" + dependencies: + - "node-gyp-cache" tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh --tflite" diff --git a/taskcluster/win-opt-base.tyml b/taskcluster/win-opt-base.tyml index 14ef60bb..f1a3c680 100644 --- a/taskcluster/win-opt-base.tyml +++ b/taskcluster/win-opt-base.tyml @@ -34,6 +34,10 @@ payload: sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c url: >- https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download + - directory: .node-gyp + format: tar.gz + content: + url: ${system.node_gyp_cache} env: TC_MSYS_VERSION: 'MSYS_NT-6.3'