diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 7e9809e5..fee2c1d6 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -36,3 +36,7 @@ build: gradle_cache: url: '' namespace: '' + build_or_cache: + repo: "${event.head.repo.url}" + sha: "${event.head.sha}" + dir: "DeepSpeech/ds" diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index dadb1094..b04ff9de 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -119,6 +119,16 @@ system: swig: repo: "https://github.com/lissyx/swig" sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118" + swig_build: + linux: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" + namespace: "project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" + osx: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" + namespace: "project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" + win: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" + namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" username: 'build-user' homedir: linux: '/home/build-user' diff --git a/taskcluster/android-cache-arm64-v8a-android-24.yml b/taskcluster/android-cache-arm64-v8a-android-24.yml index a8fef42f..8fa8e927 100644 --- a/taskcluster/android-cache-arm64-v8a-android-24.yml +++ b/taskcluster/android-cache-arm64-v8a-android-24.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.arm64_v8a.android_24.url} - namespace: ${system.android_cache.arm64_v8a.android_24.namespace} + artifact_url: ${system.android_cache.arm64_v8a.android_24.url} + artifact_namespace: ${system.android_cache.arm64_v8a.android_24.namespace} scripts: build: "taskcluster/android_cache-build.sh arm64-v8a android-24" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-arm64-v8a-android-25.yml b/taskcluster/android-cache-arm64-v8a-android-25.yml index 7dca05ea..36638726 100644 --- a/taskcluster/android-cache-arm64-v8a-android-25.yml +++ b/taskcluster/android-cache-arm64-v8a-android-25.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.arm64_v8a.android_25.url} - namespace: ${system.android_cache.arm64_v8a.android_25.namespace} + artifact_url: ${system.android_cache.arm64_v8a.android_25.url} + artifact_namespace: ${system.android_cache.arm64_v8a.android_25.namespace} scripts: build: "taskcluster/android_cache-build.sh arm64-v8a android-25" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-armeabi-v7a-android-24.yml b/taskcluster/android-cache-armeabi-v7a-android-24.yml index 605f0e92..3536ab3f 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-24.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-24.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.armeabi_v7a.android_24.url} - namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} + artifact_url: ${system.android_cache.armeabi_v7a.android_24.url} + artifact_namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} scripts: build: "taskcluster/android_cache-build.sh armeabi-v7a android-24" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-armeabi-v7a-android-25.yml b/taskcluster/android-cache-armeabi-v7a-android-25.yml index 6089d940..e4f78de7 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-25.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-25.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.armeabi_v7a.android_25.url} - namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} + artifact_url: ${system.android_cache.armeabi_v7a.android_25.url} + artifact_namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} scripts: build: "taskcluster/android_cache-build.sh armeabi-v7a android-25" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-sdk-android-27.yml b/taskcluster/android-cache-sdk-android-27.yml index 7632e5b8..668e142e 100644 --- a/taskcluster/android-cache-sdk-android-27.yml +++ b/taskcluster/android-cache-sdk-android-27.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.sdk.android_27.url} - namespace: ${system.android_cache.sdk.android_27.namespace} + artifact_url: ${system.android_cache.sdk.android_27.url} + artifact_namespace: ${system.android_cache.sdk.android_27.namespace} scripts: build: "taskcluster/android_cache-build.sh sdk android-27" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-24.yml b/taskcluster/android-cache-x86_64-android-24.yml index 38013d08..ff06f551 100644 --- a/taskcluster/android-cache-x86_64-android-24.yml +++ b/taskcluster/android-cache-x86_64-android-24.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_24.url} - namespace: ${system.android_cache.x86_64.android_24.namespace} + artifact_url: ${system.android_cache.x86_64.android_24.url} + artifact_namespace: ${system.android_cache.x86_64.android_24.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-24" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-25.yml b/taskcluster/android-cache-x86_64-android-25.yml index 6f57be42..8ccac4f9 100644 --- a/taskcluster/android-cache-x86_64-android-25.yml +++ b/taskcluster/android-cache-x86_64-android-25.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_25.url} - namespace: ${system.android_cache.x86_64.android_25.namespace} + artifact_url: ${system.android_cache.x86_64.android_25.url} + artifact_namespace: ${system.android_cache.x86_64.android_25.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-25" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-26.yml b/taskcluster/android-cache-x86_64-android-26.yml index ec711ede..0a74e78e 100644 --- a/taskcluster/android-cache-x86_64-android-26.yml +++ b/taskcluster/android-cache-x86_64-android-26.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_26.url} - namespace: ${system.android_cache.x86_64.android_26.namespace} + artifact_url: ${system.android_cache.x86_64.android_26.url} + artifact_namespace: ${system.android_cache.x86_64.android_26.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-26" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-28.yml b/taskcluster/android-cache-x86_64-android-28.yml index 471f33b9..4fa124fd 100644 --- a/taskcluster/android-cache-x86_64-android-28.yml +++ b/taskcluster/android-cache-x86_64-android-28.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_28.url} - namespace: ${system.android_cache.x86_64.android_28.namespace} + artifact_url: ${system.android_cache.x86_64.android_28.url} + artifact_namespace: ${system.android_cache.x86_64.android_28.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-28" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android-cache-x86_64-android-29.yml b/taskcluster/android-cache-x86_64-android-29.yml index 835453f9..57c194f2 100644 --- a/taskcluster/android-cache-x86_64-android-29.yml +++ b/taskcluster/android-cache-x86_64-android-29.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > ${java.packages_trusty.apt} cache: - url: ${system.android_cache.x86_64.android_29.url} - namespace: ${system.android_cache.x86_64.android_29.namespace} + artifact_url: ${system.android_cache.x86_64.android_29.url} + artifact_namespace: ${system.android_cache.x86_64.android_29.namespace} scripts: build: "taskcluster/android_cache-build.sh x86_64 android-29" package: "taskcluster/android_cache-package.sh" diff --git a/taskcluster/android_cache-opt-base.tyml b/taskcluster/android_cache-opt-base.tyml deleted file mode 100644 index b44778aa..00000000 --- a/taskcluster/android_cache-opt-base.tyml +++ /dev/null @@ -1,52 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerType} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - image: ${build.docker_image} - - features: - taskclusterProxy: true - - command: - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - extraSystemSetup: { $eval: strip(str(build.system_setup)) } - taskIndexExpire: { $fromNow: '6 months' } - in: > - (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && - apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; - cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.url}` && - if [ "$cache_file" != "200" ]; then - ${extraSystemSetup} && - adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && - mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && - echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${build.cache.namespace} - fi; - - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/android_cache-package.sh b/taskcluster/android_cache-package.sh index 22ec767d..8c73070d 100755 --- a/taskcluster/android_cache-package.sh +++ b/taskcluster/android_cache-package.sh @@ -2,17 +2,6 @@ set -xe -TC_EXPIRE=$1 -TC_INSTANCE=$2 -TC_INDEX=$3 - source $(dirname "$0")/tc-tests-utils.sh cd $HOME/ && tar -czf $TASKCLUSTER_ARTIFACTS/android_cache.tar.gz DeepSpeech/Android/ - -if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ - "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" -fi; diff --git a/taskcluster/pyenv-darwin-opt-base.tyml b/taskcluster/generic_tc_caching-darwin-opt-base.tyml similarity index 66% rename from taskcluster/pyenv-darwin-opt-base.tyml rename to taskcluster/generic_tc_caching-darwin-opt-base.tyml index 063c6613..e6777f22 100644 --- a/taskcluster/pyenv-darwin-opt-base.tyml +++ b/taskcluster/generic_tc_caching-darwin-opt-base.tyml @@ -35,12 +35,13 @@ payload: export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && env && mkdir -p $TASKCLUSTER_ARTIFACTS/ && - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.osx.url}` && - if [ "$swig_bin" != "200" ]; then - git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && - cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} 127.0.0.1:8080 ${system.pyenv.osx.namespace} + cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` && + if [ "$cache_artifact" != "200" ]; then + git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && + cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && + $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && + $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && + $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} 127.0.0.1:8080 ${build.cache.artifact_namespace} fi; artifacts: diff --git a/taskcluster/generic_tc_caching-linux-opt-base.tyml b/taskcluster/generic_tc_caching-linux-opt-base.tyml new file mode 100644 index 00000000..36668a0c --- /dev/null +++ b/taskcluster/generic_tc_caching-linux-opt-base.tyml @@ -0,0 +1,50 @@ +taskId: ${taskcluster.taskId} +provisionerId: ${taskcluster.docker.provisionerId} +workerType: ${taskcluster.docker.workerType} +taskGroupId: ${taskcluster.taskGroupId} +schedulerId: ${taskcluster.schedulerId} +created: { $fromNow: '0 sec' } +deadline: { $fromNow: '1 day' } +expires: { $fromNow: '6 months' } +scopes: + - "index:insert-task:project.deepspeech.*" + +payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + image: ${build.docker_image} + + features: + taskclusterProxy: true + + command: + - "/bin/bash" + - "--login" + - "-cxe" + - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + taskIndexExpire: { $fromNow: '6 months' } + in: > + (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && + apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; + cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` && + if [ "$cache_artifact" != "200" ]; then + adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && + mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && + echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${build.build_or_cache.repo} ~/${build.build_or_cache.dir}/ && cd ~/${build.build_or_cache.dir}/ && git checkout --quiet ${build.build_or_cache.sha})" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.build} && + sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.package} && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace} + fi; + + artifacts: + "public": + type: "directory" + path: "/tmp/artifacts/" + expires: { $fromNow: '6 months' } + +metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/generic_tc_caching-win-opt-base.tyml b/taskcluster/generic_tc_caching-win-opt-base.tyml new file mode 100644 index 00000000..a4110318 --- /dev/null +++ b/taskcluster/generic_tc_caching-win-opt-base.tyml @@ -0,0 +1,57 @@ +taskId: ${taskcluster.taskId} +provisionerId: ${taskcluster.docker.provisionerId} +workerType: ${taskcluster.docker.workerTypeWin} +taskGroupId: ${taskcluster.taskGroupId} +schedulerId: ${taskcluster.schedulerId} +created: { $fromNow: '0 sec' } +deadline: { $fromNow: '1 day' } +expires: { $fromNow: '6 months' } +scopes: + - "index:insert-task:project.deepspeech.*" + +payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + + features: + taskclusterProxy: true + + mounts: + - file: msys2-base-x86_64.tar.xz + content: + sha256: c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60 + url: >- + https://github.com/msys2/msys2-installer/releases/download/2020-05-17/msys2-base-x86_64-20200517.tar.xz + + env: + TC_MSYS_VERSION: 'MSYS_NT-6.3-9600' + MSYS: 'winsymlinks:nativestrict' + + command: + - >- + "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | + "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si + - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0" + - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" + - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" + - $let: + taskIndexExpire: { $fromNow: '6 months' } + in: > + echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C && + export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && + export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" && + export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && + echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh && + env && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k + + - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE" + + artifacts: + - type: "directory" + path: "public/" + expires: { $fromNow: '6 months' } + +metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/gradle-cache.yml b/taskcluster/gradle-cache.yml index 334545c3..ef589197 100644 --- a/taskcluster/gradle-cache.yml +++ b/taskcluster/gradle-cache.yml @@ -1,11 +1,11 @@ build: - template_file: android_cache-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.gradle_cache.url} + artifact_namespace: ${system.gradle_cache.namespace} system_setup: > ${java.packages_trusty.apt} - cache: - url: ${system.gradle_cache.url} - namespace: ${system.gradle_cache.namespace} scripts: build: "taskcluster/gradle-build.sh" package: "taskcluster/gradle-package.sh" diff --git a/taskcluster/gradle-package.sh b/taskcluster/gradle-package.sh index 495c05e8..840fffc8 100755 --- a/taskcluster/gradle-package.sh +++ b/taskcluster/gradle-package.sh @@ -2,17 +2,6 @@ set -xe -TC_EXPIRE=$1 -TC_INSTANCE=$2 -TC_INDEX=$3 - source $(dirname "$0")/tc-tests-utils.sh cd ${GRADLE_USER_HOME}/../ && tar -czf $TASKCLUSTER_ARTIFACTS/gradle.tar.gz gradle-cache/ - -if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ - "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" -fi; diff --git a/taskcluster/homebrew-darwin-opt-base.tyml b/taskcluster/homebrew-darwin-opt-base.tyml deleted file mode 100644 index ac6c5af3..00000000 --- a/taskcluster/homebrew-darwin-opt-base.tyml +++ /dev/null @@ -1,59 +0,0 @@ -taskId: ${taskcluster.taskId} -provisionerId: ${taskcluster.generic.provisionerId} -workerType: ${taskcluster.generic.workerType} -taskGroupId: ${taskcluster.taskGroupId} -schedulerId: ${taskcluster.schedulerId} -dependencies: - $map: { $eval: build.dependencies } - each(b): - $eval: as_slugid(b) -created: { $fromNow: '0 sec' } -deadline: { $fromNow: '1 day' } -expires: { $fromNow: '6 months' } -scopes: - - "index:insert-task:project.deepspeech.*" - -payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - - features: - taskclusterProxy: true - - command: - - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - taskIndexExpire: { $fromNow: '6 months' } - in: > - export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && - export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && - (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && - export TASKCLUSTER_TASK_DIR="$(pwd)" && - export LC_ALL=C && - export MACOSX_DEPLOYMENT_TARGET=10.10 && - export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && - env && - mkdir -p $TASKCLUSTER_ARTIFACTS/ && - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.homebrew.url}` && - if [ "$swig_bin" != "200" ]; then - git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && - cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && - $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} && - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ - "http://127.0.0.1:8080/index/v1/task/${build.homebrew.namespace}" - fi; - - artifacts: - - type: "directory" - path: "public/" - expires: { $fromNow: '6 months' } - -metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/homebrew_builds-darwin-amd64.yml b/taskcluster/homebrew_builds-darwin-amd64.yml index 4dac3a13..3377dcac 100644 --- a/taskcluster/homebrew_builds-darwin-amd64.yml +++ b/taskcluster/homebrew_builds-darwin-amd64.yml @@ -1,8 +1,8 @@ build: - template_file: homebrew-darwin-opt-base.tyml - homebrew: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz' - namespace: 'project.deepspeech.homebrew_builds.5' + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.homebrew_builds.url} + artifact_namespace: ${system.homebrew_builds.namespace} scripts: build: "taskcluster/homebrew-build.sh --builds" package: "taskcluster/homebrew-package.sh --builds" diff --git a/taskcluster/homebrew_tests-darwin-amd64.yml b/taskcluster/homebrew_tests-darwin-amd64.yml index fc9637f2..dc93c183 100644 --- a/taskcluster/homebrew_tests-darwin-amd64.yml +++ b/taskcluster/homebrew_tests-darwin-amd64.yml @@ -1,8 +1,8 @@ build: - template_file: homebrew-darwin-opt-base.tyml - homebrew: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.6/artifacts/public/homebrew_tests.tar.gz' - namespace: 'project.deepspeech.homebrew_tests.6' + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.homebrew_tests.url} + artifact_namespace: ${system.homebrew_tests.namespace} scripts: build: "taskcluster/homebrew-build.sh --tests" package: "taskcluster/homebrew-package.sh --tests" diff --git a/taskcluster/node-gyp-cache-base.tyml b/taskcluster/node-gyp-cache-base.tyml deleted file mode 100644 index 5c0e895d..00000000 --- a/taskcluster/node-gyp-cache-base.tyml +++ /dev/null @@ -1,57 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerType} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "node:12" - - features: - taskclusterProxy: true - - # This task will inspect system.node_gyp_cache taskcluster index existence: - # - if the artifact does not exists, it will build it - # - if the artifact exists, it will re-mirror it (if we don't do that, new - # index gets published with no artifact and erases existing one) - command: - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - extraSystemSetup: { $eval: strip(str(build.system_setup)) } - taskIndexExpire: { $fromNow: '6 months' } - in: > - apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; - cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache.url}` && - if [ "$cache_file" != "200" ]; then - mkdir -p ~/DeepSpeech/ds/ && - git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && - cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && - ~/DeepSpeech/ds/${build.scripts.build} && - ~/DeepSpeech/ds/${build.scripts.package} && - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ - "http://taskcluster/index/v1/task/${system.node_gyp_cache.namespace}" - fi; - - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/node-gyp-cache.yml b/taskcluster/node-gyp-cache.yml index a9be39de..a0c48ba5 100644 --- a/taskcluster/node-gyp-cache.yml +++ b/taskcluster/node-gyp-cache.yml @@ -1,5 +1,12 @@ build: - template_file: node-gyp-cache-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml + docker_image: "node:12" + cache: + artifact_url: "${system.node_gyp_cache.url}" + artifact_namespace: "${system.node_gyp_cache.namespace}" + system_setup: + > + (apt-get -qq -y install sudo || true) scripts: build: "taskcluster/node-gyp-populate.sh" package: "taskcluster/node-gyp-package.sh" diff --git a/taskcluster/node-gyp-populate.sh b/taskcluster/node-gyp-populate.sh index 7043d332..333c692a 100755 --- a/taskcluster/node-gyp-populate.sh +++ b/taskcluster/node-gyp-populate.sh @@ -8,7 +8,9 @@ node --version npm --version -npm install -g node-gyp@6.x +npm install node-gyp@6.x + +export PATH=$HOME/node_modules/.bin/:$PATH devDir=$DS_ROOT_TASK/node-gyp-cache/ diff --git a/taskcluster/pyenv-darwin-amd64.yml b/taskcluster/pyenv-darwin-amd64.yml index 47cceec9..3ad055ca 100644 --- a/taskcluster/pyenv-darwin-amd64.yml +++ b/taskcluster/pyenv-darwin-amd64.yml @@ -1,5 +1,8 @@ build: - template_file: pyenv-darwin-opt-base.tyml + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.pyenv.osx.url} + artifact_namespace: ${system.pyenv.osx.namespace} scripts: build: "taskcluster/pyenv-build.sh" package: "taskcluster/pyenv-package.sh" diff --git a/taskcluster/pyenv-linux-amd64.yml b/taskcluster/pyenv-linux-amd64.yml index 393a8872..a8ece0b4 100644 --- a/taskcluster/pyenv-linux-amd64.yml +++ b/taskcluster/pyenv-linux-amd64.yml @@ -1,5 +1,8 @@ build: - template_file: pyenv-linux-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: "${system.pyenv.linux.url}" + artifact_namespace: "${system.pyenv.linux.namespace}" system_setup: > apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_trusty.apt} wget diff --git a/taskcluster/pyenv-linux-opt-base.tyml b/taskcluster/pyenv-linux-opt-base.tyml deleted file mode 100644 index 15d8f7b0..00000000 --- a/taskcluster/pyenv-linux-opt-base.tyml +++ /dev/null @@ -1,52 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerType} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - image: ${build.docker_image} - - features: - taskclusterProxy: true - - command: - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - extraSystemSetup: { $eval: strip(str(build.system_setup)) } - taskIndexExpire: { $fromNow: '6 months' } - in: > - (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && - apt-get -qq update && apt-get -qq -y install curl git; - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.linux.url}` && - if [ "$swig_bin" != "200" ]; then - ${extraSystemSetup} && - adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && - mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && - echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.linux.namespace} - fi; - - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/pyenv-package.sh b/taskcluster/pyenv-package.sh index bc2c0639..73f6a542 100755 --- a/taskcluster/pyenv-package.sh +++ b/taskcluster/pyenv-package.sh @@ -2,17 +2,6 @@ set -xe -TC_EXPIRE=$1 -TC_INSTANCE=$2 -TC_INDEX=$3 - source $(dirname "$0")/tc-tests-utils.sh cd ${PYENV_ROOT}/ && $TAR -czf $TASKCLUSTER_ARTIFACTS/pyenv.tar.gz . - -if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ - "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" -fi; diff --git a/taskcluster/pyenv-win-amd64.yml b/taskcluster/pyenv-win-amd64.yml index d22872b3..7d05f141 100644 --- a/taskcluster/pyenv-win-amd64.yml +++ b/taskcluster/pyenv-win-amd64.yml @@ -1,5 +1,8 @@ build: - template_file: pyenv-win-opt-base.tyml + template_file: generic_tc_caching-win-opt-base.tyml + cache: + artifact_url: "${system.pyenv.win.url}" + artifact_namespace: "${system.pyenv.win.namespace}" scripts: build: "taskcluster/pyenv-build.sh" package: "taskcluster/pyenv-package.sh" diff --git a/taskcluster/pyenv-win-opt-base.tyml b/taskcluster/pyenv-win-opt-base.tyml deleted file mode 100644 index 59c584ba..00000000 --- a/taskcluster/pyenv-win-opt-base.tyml +++ /dev/null @@ -1,59 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerTypeWin} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - - features: - taskclusterProxy: true - - mounts: - - file: msys2-base-x86_64.tar.xz - content: - sha256: c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60 - url: >- - https://github.com/msys2/msys2-installer/releases/download/2020-05-17/msys2-base-x86_64-20200517.tar.xz - - env: - TC_MSYS_VERSION: 'MSYS_NT-6.3-9600' - MSYS: 'winsymlinks:nativestrict' - - command: - - >- - "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | - "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si - - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0" - - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - - $let: - taskIndexExpire: { $fromNow: '6 months' } - in: > - echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C && - export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && - export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" && - export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && - echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh && - env && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${system.pyenv.win.url}`\" != \"200\" ]; then git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.win.namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k - - - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE" - - artifacts: - - type: "directory" - path: "public/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/swig-darwin-amd64.yml b/taskcluster/swig-darwin-amd64.yml index cb08cdef..64ca794b 100644 --- a/taskcluster/swig-darwin-amd64.yml +++ b/taskcluster/swig-darwin-amd64.yml @@ -1,7 +1,12 @@ build: - template_file: swig-darwin-opt-base.tyml - swig_system: "darwin" - swig_arch: "amd64" + template_file: generic_tc_caching-darwin-opt-base.tyml + build_or_cache: + repo: "${system.swig.repo}" + sha: "${system.swig.sha1}" + dir: "swig" + cache: + artifact_url: "${system.swig_build.osx.url}" + artifact_namespace: "${system.swig_build.osx.namespace}" scripts: build: "taskcluster/build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/swig-darwin-opt-base.tyml b/taskcluster/swig-darwin-opt-base.tyml deleted file mode 100644 index 0f08e596..00000000 --- a/taskcluster/swig-darwin-opt-base.tyml +++ /dev/null @@ -1,56 +0,0 @@ -taskId: ${taskcluster.taskId} -provisionerId: ${taskcluster.generic.provisionerId} -workerType: ${taskcluster.generic.workerType} -taskGroupId: ${taskcluster.taskGroupId} -schedulerId: ${taskcluster.schedulerId} -dependencies: - $map: { $eval: build.dependencies } - each(b): - $eval: as_slugid(b) -created: { $fromNow: '0 sec' } -deadline: { $fromNow: '1 day' } -expires: { $fromNow: '6 months' } -scopes: - - "index:insert-task:project.deepspeech.*" - -payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - - command: - - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - taskIndexExpire: { $fromNow: '6 months' } - in: > - export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && - export TASKCLUSTER_TASKDIR="$(pwd)" && - export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && - export LC_ALL=C && - export MACOSX_DEPLOYMENT_TARGET=10.10 && - export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && - export HOMEBREW_NO_AUTO_UPDATE=1 && - env && - mkdir -p $TASKCLUSTER_ARTIFACTS/ && - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` && - if [ "$swig_bin" != "200" ]; then - git clone --quiet ${system.swig.repo} $TASKCLUSTER_TASKDIR/swig/ && - cd $TASKCLUSTER_TASKDIR/swig/ && git checkout --quiet ${system.swig.sha1} && - $TASKCLUSTER_TASKDIR/swig/${build.scripts.build} && - $TASKCLUSTER_TASKDIR/swig/${build.scripts.package} && - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ - "http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}" - fi; - - artifacts: - - type: "directory" - path: "public/" - expires: { $fromNow: '6 months' } - -metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/swig-linux-amd64.yml b/taskcluster/swig-linux-amd64.yml index 27bd058c..3b199c1a 100644 --- a/taskcluster/swig-linux-amd64.yml +++ b/taskcluster/swig-linux-amd64.yml @@ -1,8 +1,13 @@ build: - template_file: swig-linux-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml docker_image: "ubuntu:14.04" - swig_system: "linux" - swig_arch: "amd64" + build_or_cache: + repo: "${system.swig.repo}" + sha: "${system.swig.sha1}" + dir: "swig" + cache: + artifact_url: "${system.swig_build.linux.url}" + artifact_namespace: "${system.swig_build.linux.namespace}" system_setup: > apt-get -qq -y install autoconf automake bison build-essential diff --git a/taskcluster/swig-linux-opt-base.tyml b/taskcluster/swig-linux-opt-base.tyml deleted file mode 100644 index fb3662e4..00000000 --- a/taskcluster/swig-linux-opt-base.tyml +++ /dev/null @@ -1,54 +0,0 @@ -$if: 'event.event in build.allowed' -then: - taskId: ${taskcluster.taskId} - provisionerId: ${taskcluster.docker.provisionerId} - workerType: ${taskcluster.docker.workerType} - taskGroupId: ${taskcluster.taskGroupId} - schedulerId: ${taskcluster.schedulerId} - created: { $fromNow: '0 sec' } - deadline: { $fromNow: '1 day' } - expires: { $fromNow: '6 months' } - scopes: - - "index:insert-task:project.deepspeech.*" - - payload: - maxRunTime: { $eval: to_int(build.maxRunTime) } - image: ${build.docker_image} - - features: - taskclusterProxy: true - - command: - - "/bin/bash" - - "--login" - - "-cxe" - - $let: - extraSystemSetup: { $eval: strip(str(build.system_setup)) } - taskIndexExpire: { $fromNow: '6 months' } - in: > - (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && - apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; - swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` && - if [ "$swig_bin" != "200" ]; then - adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${system.swig.repo} ~/swig/ && cd ~/swig/ && git checkout --quiet ${system.swig.sha1})" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && - sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/swig/${build.scripts.build} && - sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/swig/${build.scripts.package} && - curl -sSL --fail -X PUT \ - -H "Content-Type: application/json" \ - -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ - "http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}" - fi; - - artifacts: - "public": - type: "directory" - path: "/tmp/artifacts/" - expires: { $fromNow: '6 months' } - - metadata: - name: ${build.metadata.name} - description: ${build.metadata.description} - owner: ${event.head.user.email} - source: ${event.head.repo.url} diff --git a/taskcluster/swig-win-amd64.yml b/taskcluster/swig-win-amd64.yml index 576bdf73..310a8112 100644 --- a/taskcluster/swig-win-amd64.yml +++ b/taskcluster/swig-win-amd64.yml @@ -1,8 +1,13 @@ build: - template_file: swig-linux-opt-base.tyml + template_file: generic_tc_caching-linux-opt-base.tyml docker_image: "ubuntu:18.04" - swig_system: "win" - swig_arch: "amd64" + build_or_cache: + repo: "${system.swig.repo}" + sha: "${system.swig.sha1}" + dir: "swig" + cache: + artifact_url: "${system.swig_build.win.url}" + artifact_namespace: "${system.swig_build.win.namespace}" system_setup: > apt-get -qq -y install autoconf automake bison build-essential mingw-w64 && diff --git a/taskcluster/tc-update-index.sh b/taskcluster/tc-update-index.sh new file mode 100755 index 00000000..fbc72316 --- /dev/null +++ b/taskcluster/tc-update-index.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Helper script because it is way too painful to deal with Windows' CMD.exe +# ways of escaping things when pushing JSON + +set -xe + +TC_EXPIRE=$1 +TC_INSTANCE=$2 +TC_INDEX=$3 + +source $(dirname "$0")/tc-tests-utils.sh + +if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then + curl -sSL --fail -X PUT \ + -H "Content-Type: application/json" \ + -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \ + "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}" +fi;