Fix #3068: More generic TaskCluster build/caching tasks

taskcluster/.build.yml

@@ -36,3 +36,7 @@ build:
   gradle_cache:
     url: ''
     namespace: ''
+  build_or_cache:
+    repo: "${event.head.repo.url}"
+    sha: "${event.head.sha}"
+    dir: "DeepSpeech/ds"

taskcluster/.shared.yml

@@ -119,6 +119,16 @@ system:
   swig:
     repo: "https://github.com/lissyx/swig"
     sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
+  swig_build:
+    linux:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz"
+      namespace: "project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
+    osx:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz"
+      namespace: "project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
+    win:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz"
+      namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
   username: 'build-user'
   homedir:
     linux: '/home/build-user'

taskcluster/android-cache-arm64-v8a-android-24.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.arm64_v8a.android_24.url}
+    artifact_url: ${system.android_cache.arm64_v8a.android_24.url}
-    namespace: ${system.android_cache.arm64_v8a.android_24.namespace}
+    artifact_namespace: ${system.android_cache.arm64_v8a.android_24.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh arm64-v8a android-24"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-arm64-v8a-android-25.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.arm64_v8a.android_25.url}
+    artifact_url: ${system.android_cache.arm64_v8a.android_25.url}
-    namespace: ${system.android_cache.arm64_v8a.android_25.namespace}
+    artifact_namespace: ${system.android_cache.arm64_v8a.android_25.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh arm64-v8a android-25"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-armeabi-v7a-android-24.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.armeabi_v7a.android_24.url}
+    artifact_url: ${system.android_cache.armeabi_v7a.android_24.url}
-    namespace: ${system.android_cache.armeabi_v7a.android_24.namespace}
+    artifact_namespace: ${system.android_cache.armeabi_v7a.android_24.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh armeabi-v7a android-24"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-armeabi-v7a-android-25.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.armeabi_v7a.android_25.url}
+    artifact_url: ${system.android_cache.armeabi_v7a.android_25.url}
-    namespace: ${system.android_cache.armeabi_v7a.android_25.namespace}
+    artifact_namespace: ${system.android_cache.armeabi_v7a.android_25.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh armeabi-v7a android-25"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-sdk-android-27.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.sdk.android_27.url}
+    artifact_url: ${system.android_cache.sdk.android_27.url}
-    namespace: ${system.android_cache.sdk.android_27.namespace}
+    artifact_namespace: ${system.android_cache.sdk.android_27.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh sdk android-27"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-x86_64-android-24.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.x86_64.android_24.url}
+    artifact_url: ${system.android_cache.x86_64.android_24.url}
-    namespace: ${system.android_cache.x86_64.android_24.namespace}
+    artifact_namespace: ${system.android_cache.x86_64.android_24.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh x86_64 android-24"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-x86_64-android-25.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.x86_64.android_25.url}
+    artifact_url: ${system.android_cache.x86_64.android_25.url}
-    namespace: ${system.android_cache.x86_64.android_25.namespace}
+    artifact_namespace: ${system.android_cache.x86_64.android_25.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh x86_64 android-25"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-x86_64-android-26.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.x86_64.android_26.url}
+    artifact_url: ${system.android_cache.x86_64.android_26.url}
-    namespace: ${system.android_cache.x86_64.android_26.namespace}
+    artifact_namespace: ${system.android_cache.x86_64.android_26.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh x86_64 android-26"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-x86_64-android-28.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.x86_64.android_28.url}
+    artifact_url: ${system.android_cache.x86_64.android_28.url}
-    namespace: ${system.android_cache.x86_64.android_28.namespace}
+    artifact_namespace: ${system.android_cache.x86_64.android_28.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh x86_64 android-28"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android-cache-x86_64-android-29.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   system_setup:
     >
       ${java.packages_trusty.apt}
   cache:
-    url: ${system.android_cache.x86_64.android_29.url}
+    artifact_url: ${system.android_cache.x86_64.android_29.url}
-    namespace: ${system.android_cache.x86_64.android_29.namespace}
+    artifact_namespace: ${system.android_cache.x86_64.android_29.namespace}
   scripts:
     build: "taskcluster/android_cache-build.sh x86_64 android-29"
     package: "taskcluster/android_cache-package.sh"

taskcluster/android_cache-opt-base.tyml

@@ -1,52 +0,0 @@
-$if: 'event.event in build.allowed'
-then:
-  taskId: ${taskcluster.taskId}
-  provisionerId: ${taskcluster.docker.provisionerId}
-  workerType: ${taskcluster.docker.workerType}
-  taskGroupId: ${taskcluster.taskGroupId}
-  schedulerId: ${taskcluster.schedulerId}
-  created: { $fromNow: '0 sec' }
-  deadline: { $fromNow: '1 day' }
-  expires: { $fromNow: '6 months' }
-  scopes:
-    - "index:insert-task:project.deepspeech.*"
-
-  payload:
-    maxRunTime: { $eval: to_int(build.maxRunTime) }
-    image: ${build.docker_image}
-
-    features:
-      taskclusterProxy: true
-
-    command:
-      - "/bin/bash"
-      - "--login"
-      - "-cxe"
-      - $let:
-          extraSystemSetup: { $eval: strip(str(build.system_setup)) }
-          taskIndexExpire: { $fromNow: '6 months' }
-        in: >
-          (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
-          apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
-          cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.url}` &&
-          if [ "$cache_file" != "200" ]; then
-            ${extraSystemSetup} &&
-            adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ &&
-            mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts &&
-            echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
-            sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
-            sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
-            sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${build.cache.namespace}
-          fi;
-
-    artifacts:
-      "public":
-        type: "directory"
-        path: "/tmp/artifacts/"
-        expires: { $fromNow: '6 months' }
-
-  metadata:
-    name: ${build.metadata.name}
-    description: ${build.metadata.description}
-    owner: ${event.head.user.email}
-    source: ${event.head.repo.url}

taskcluster/android_cache-package.sh

@@ -2,17 +2,6 @@
 
 set -xe
 
-TC_EXPIRE=$1
-TC_INSTANCE=$2
-TC_INDEX=$3
-
 source $(dirname "$0")/tc-tests-utils.sh
 
 cd $HOME/ && tar -czf $TASKCLUSTER_ARTIFACTS/android_cache.tar.gz DeepSpeech/Android/
-
-if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then
-    curl -sSL --fail -X PUT \
-        -H "Content-Type: application/json" \
-        -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \
-        "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}"
-fi;

taskcluster/generic_tc_caching-darwin-opt-base.tyml

@@ -35,12 +35,13 @@ payload:
           export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
           env &&
           mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
-          swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.osx.url}` &&
+          cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` &&
-          if [ "$swig_bin" != "200" ]; then
+          if [ "$cache_artifact" != "200" ]; then
-            git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
+            git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} &&
-            cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
+            cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} &&
-            $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
+            $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} &&
-            $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} 127.0.0.1:8080 ${system.pyenv.osx.namespace}
+            $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} &&
+            $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} 127.0.0.1:8080 ${build.cache.artifact_namespace}
           fi;
 
   artifacts:

taskcluster/generic_tc_caching-linux-opt-base.tyml

@@ -0,0 +1,50 @@
+taskId: ${taskcluster.taskId}
+provisionerId: ${taskcluster.docker.provisionerId}
+workerType: ${taskcluster.docker.workerType}
+taskGroupId: ${taskcluster.taskGroupId}
+schedulerId: ${taskcluster.schedulerId}
+created: { $fromNow: '0 sec' }
+deadline: { $fromNow: '1 day' }
+expires: { $fromNow: '6 months' }
+scopes:
+  - "index:insert-task:project.deepspeech.*"
+
+payload:
+  maxRunTime: { $eval: to_int(build.maxRunTime) }
+  image: ${build.docker_image}
+
+  features:
+    taskclusterProxy: true
+
+  command:
+    - "/bin/bash"
+    - "--login"
+    - "-cxe"
+    - $let:
+        extraSystemSetup: { $eval: strip(str(build.system_setup)) }
+        taskIndexExpire: { $fromNow: '6 months' }
+      in: >
+        (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
+        apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
+        cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` &&
+        if [ "$cache_artifact" != "200" ]; then
+          adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ &&
+          mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${build.build_or_cache.repo} ~/${build.build_or_cache.dir}/ && cd ~/${build.build_or_cache.dir}/ && git checkout --quiet ${build.build_or_cache.sha})" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
+          sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.build} &&
+          sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.package} &&
+          sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}
+        fi;
+
+  artifacts:
+    "public":
+      type: "directory"
+      path: "/tmp/artifacts/"
+      expires: { $fromNow: '6 months' }
+
+metadata:
+  name: ${build.metadata.name}
+  description: ${build.metadata.description}
+  owner: ${event.head.user.email}
+  source: ${event.head.repo.url}

taskcluster/generic_tc_caching-win-opt-base.tyml

@@ -0,0 +1,57 @@
+taskId: ${taskcluster.taskId}
+provisionerId: ${taskcluster.docker.provisionerId}
+workerType: ${taskcluster.docker.workerTypeWin}
+taskGroupId: ${taskcluster.taskGroupId}
+schedulerId: ${taskcluster.schedulerId}
+created: { $fromNow: '0 sec' }
+deadline: { $fromNow: '1 day' }
+expires: { $fromNow: '6 months' }
+scopes:
+  - "index:insert-task:project.deepspeech.*"
+
+payload:
+  maxRunTime: { $eval: to_int(build.maxRunTime) }
+
+  features:
+    taskclusterProxy: true
+
+  mounts:
+    - file: msys2-base-x86_64.tar.xz
+      content:
+        sha256: c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60
+        url: >-
+          https://github.com/msys2/msys2-installer/releases/download/2020-05-17/msys2-base-x86_64-20200517.tar.xz
+
+  env:
+    TC_MSYS_VERSION: 'MSYS_NT-6.3-9600'
+    MSYS: 'winsymlinks:nativestrict'
+
+  command:
+    - >-
+      "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
+      "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
+    - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
+    - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
+    - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
+    - $let:
+        taskIndexExpire: { $fromNow: '6 months' }
+      in: >
+        echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C &&
+        export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" &&
+        export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" &&
+        export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
+        echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh &&
+        env && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k
+
+    - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE"
+
+  artifacts:
+    - type: "directory"
+      path: "public/"
+      expires: { $fromNow: '6 months' }
+
+metadata:
+  name: ${build.metadata.name}
+  description: ${build.metadata.description}
+  owner: ${event.head.user.email}
+  source: ${event.head.repo.url}

taskcluster/gradle-cache.yml

@@ -1,11 +1,11 @@
 build:
-  template_file: android_cache-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  cache:
+    artifact_url: ${system.gradle_cache.url}
+    artifact_namespace: ${system.gradle_cache.namespace}
   system_setup:
     >
       ${java.packages_trusty.apt}
-  cache:
-    url: ${system.gradle_cache.url}
-    namespace: ${system.gradle_cache.namespace}
   scripts:
     build: "taskcluster/gradle-build.sh"
     package: "taskcluster/gradle-package.sh"

taskcluster/gradle-package.sh

@@ -2,17 +2,6 @@
 
 set -xe
 
-TC_EXPIRE=$1
-TC_INSTANCE=$2
-TC_INDEX=$3
-
 source $(dirname "$0")/tc-tests-utils.sh
 
 cd ${GRADLE_USER_HOME}/../ && tar -czf $TASKCLUSTER_ARTIFACTS/gradle.tar.gz gradle-cache/
-
-if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then
-    curl -sSL --fail -X PUT \
-        -H "Content-Type: application/json" \
-        -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \
-        "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}"
-fi;

taskcluster/homebrew-darwin-opt-base.tyml

@@ -1,59 +0,0 @@
-taskId: ${taskcluster.taskId}
-provisionerId: ${taskcluster.generic.provisionerId}
-workerType: ${taskcluster.generic.workerType}
-taskGroupId: ${taskcluster.taskGroupId}
-schedulerId: ${taskcluster.schedulerId}
-dependencies:
-  $map: { $eval: build.dependencies }
-  each(b):
-    $eval: as_slugid(b)
-created: { $fromNow: '0 sec' }
-deadline: { $fromNow: '1 day' }
-expires: { $fromNow: '6 months' }
-scopes:
-  - "index:insert-task:project.deepspeech.*"
-
-payload:
-  maxRunTime: { $eval: to_int(build.maxRunTime) }
-
-  features:
-    taskclusterProxy: true
-
-  command:
-    - - "/bin/bash"
-      - "--login"
-      - "-cxe"
-      - $let:
-          taskIndexExpire: { $fromNow: '6 months' }
-        in: >
-          export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
-          export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
-          (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
-          export TASKCLUSTER_TASK_DIR="$(pwd)" &&
-          export LC_ALL=C &&
-          export MACOSX_DEPLOYMENT_TARGET=10.10 &&
-          export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
-          env &&
-          mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
-          swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.homebrew.url}` &&
-          if [ "$swig_bin" != "200" ]; then
-            git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
-            cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
-            $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
-            $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} &&
-            curl -sSL --fail -X PUT \
-                -H "Content-Type: application/json" \
-                -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
-                "http://127.0.0.1:8080/index/v1/task/${build.homebrew.namespace}"
-          fi;
-
-  artifacts:
-    - type: "directory"
-      path: "public/"
-      expires: { $fromNow: '6 months' }
-
-metadata:
-  name: ${build.metadata.name}
-  description: ${build.metadata.description}
-  owner: ${event.head.user.email}
-  source: ${event.head.repo.url}

taskcluster/homebrew_builds-darwin-amd64.yml

@@ -1,8 +1,8 @@
 build:
-  template_file: homebrew-darwin-opt-base.tyml
+  template_file: generic_tc_caching-darwin-opt-base.tyml
-  homebrew:
+  cache:
-    url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz'
+    artifact_url: ${system.homebrew_builds.url}
-    namespace: 'project.deepspeech.homebrew_builds.5'
+    artifact_namespace: ${system.homebrew_builds.namespace}
   scripts:
     build: "taskcluster/homebrew-build.sh --builds"
     package: "taskcluster/homebrew-package.sh --builds"

taskcluster/homebrew_tests-darwin-amd64.yml

@@ -1,8 +1,8 @@
 build:
-  template_file: homebrew-darwin-opt-base.tyml
+  template_file: generic_tc_caching-darwin-opt-base.tyml
-  homebrew:
+  cache:
-    url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.6/artifacts/public/homebrew_tests.tar.gz'
+    artifact_url: ${system.homebrew_tests.url}
-    namespace: 'project.deepspeech.homebrew_tests.6'
+    artifact_namespace: ${system.homebrew_tests.namespace}
   scripts:
     build: "taskcluster/homebrew-build.sh --tests"
     package: "taskcluster/homebrew-package.sh --tests"

taskcluster/node-gyp-cache-base.tyml

@@ -1,57 +0,0 @@
-$if: 'event.event in build.allowed'
-then:
-  taskId: ${taskcluster.taskId}
-  provisionerId: ${taskcluster.docker.provisionerId}
-  workerType: ${taskcluster.docker.workerType}
-  taskGroupId: ${taskcluster.taskGroupId}
-  schedulerId: ${taskcluster.schedulerId}
-  created: { $fromNow: '0 sec' }
-  deadline: { $fromNow: '1 day' }
-  expires: { $fromNow: '6 months' }
-  scopes:
-    - "index:insert-task:project.deepspeech.*"
-
-  payload:
-    maxRunTime: { $eval: to_int(build.maxRunTime) }
-    image: "node:12"
-
-    features:
-      taskclusterProxy: true
-
-    # This task will inspect system.node_gyp_cache taskcluster index existence:
-    #  - if the artifact does not exists, it will build it
-    #  - if the artifact exists, it will re-mirror it (if we don't do that, new
-    #    index gets published with no artifact and erases existing one)
-    command:
-      - "/bin/bash"
-      - "--login"
-      - "-cxe"
-      - $let:
-          extraSystemSetup: { $eval: strip(str(build.system_setup)) }
-          taskIndexExpire: { $fromNow: '6 months' }
-        in: >
-          apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
-          cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache.url}` &&
-          if [ "$cache_file" != "200" ]; then
-            mkdir -p ~/DeepSpeech/ds/ &&
-            git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ &&
-            cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
-            ~/DeepSpeech/ds/${build.scripts.build} &&
-            ~/DeepSpeech/ds/${build.scripts.package} &&
-            curl -sSL --fail -X PUT \
-                -H "Content-Type: application/json" \
-                -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
-                "http://taskcluster/index/v1/task/${system.node_gyp_cache.namespace}"
-          fi;
-
-    artifacts:
-      "public":
-        type: "directory"
-        path: "/tmp/artifacts/"
-        expires: { $fromNow: '6 months' }
-
-  metadata:
-    name: ${build.metadata.name}
-    description: ${build.metadata.description}
-    owner: ${event.head.user.email}
-    source: ${event.head.repo.url}

taskcluster/node-gyp-cache.yml

@@ -1,5 +1,12 @@
 build:
-  template_file: node-gyp-cache-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  docker_image: "node:12"
+  cache:
+    artifact_url: "${system.node_gyp_cache.url}"
+    artifact_namespace: "${system.node_gyp_cache.namespace}"
+  system_setup:
+    >
+      (apt-get -qq -y install sudo || true)
   scripts:
     build: "taskcluster/node-gyp-populate.sh"
     package: "taskcluster/node-gyp-package.sh"

taskcluster/node-gyp-populate.sh

@@ -8,7 +8,9 @@ node --version
 
 npm --version
 
-npm install -g node-gyp@6.x
+npm install node-gyp@6.x
+
+export PATH=$HOME/node_modules/.bin/:$PATH
 
 devDir=$DS_ROOT_TASK/node-gyp-cache/
 

taskcluster/pyenv-darwin-amd64.yml

@@ -1,5 +1,8 @@
 build:
-  template_file: pyenv-darwin-opt-base.tyml
+  template_file: generic_tc_caching-darwin-opt-base.tyml
+  cache:
+    artifact_url: ${system.pyenv.osx.url}
+    artifact_namespace: ${system.pyenv.osx.namespace}
   scripts:
     build: "taskcluster/pyenv-build.sh"
     package: "taskcluster/pyenv-package.sh"

taskcluster/pyenv-linux-amd64.yml

@@ -1,5 +1,8 @@
 build:
-  template_file: pyenv-linux-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  cache:
+    artifact_url: "${system.pyenv.linux.url}"
+    artifact_namespace: "${system.pyenv.linux.namespace}"
   system_setup:
     >
       apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_trusty.apt} wget

taskcluster/pyenv-linux-opt-base.tyml

@@ -1,52 +0,0 @@
-$if: 'event.event in build.allowed'
-then:
-  taskId: ${taskcluster.taskId}
-  provisionerId: ${taskcluster.docker.provisionerId}
-  workerType: ${taskcluster.docker.workerType}
-  taskGroupId: ${taskcluster.taskGroupId}
-  schedulerId: ${taskcluster.schedulerId}
-  created: { $fromNow: '0 sec' }
-  deadline: { $fromNow: '1 day' }
-  expires: { $fromNow: '6 months' }
-  scopes:
-    - "index:insert-task:project.deepspeech.*"
-
-  payload:
-    maxRunTime: { $eval: to_int(build.maxRunTime) }
-    image: ${build.docker_image}
-
-    features:
-      taskclusterProxy: true
-
-    command:
-      - "/bin/bash"
-      - "--login"
-      - "-cxe"
-      - $let:
-          extraSystemSetup: { $eval: strip(str(build.system_setup)) }
-          taskIndexExpire: { $fromNow: '6 months' }
-        in: >
-          (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
-          apt-get -qq update && apt-get -qq -y install curl git;
-          swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.pyenv.linux.url}` &&
-          if [ "$swig_bin" != "200" ]; then
-            ${extraSystemSetup} &&
-            adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ &&
-            mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts &&
-            echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
-            sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
-            sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
-            sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.linux.namespace}
-          fi;
-
-    artifacts:
-      "public":
-        type: "directory"
-        path: "/tmp/artifacts/"
-        expires: { $fromNow: '6 months' }
-
-  metadata:
-    name: ${build.metadata.name}
-    description: ${build.metadata.description}
-    owner: ${event.head.user.email}
-    source: ${event.head.repo.url}

taskcluster/pyenv-package.sh

@@ -2,17 +2,6 @@
 
 set -xe
 
-TC_EXPIRE=$1
-TC_INSTANCE=$2
-TC_INDEX=$3
-
 source $(dirname "$0")/tc-tests-utils.sh
 
 cd ${PYENV_ROOT}/ && $TAR -czf $TASKCLUSTER_ARTIFACTS/pyenv.tar.gz .
-
-if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then
-    curl -sSL --fail -X PUT \
-        -H "Content-Type: application/json" \
-        -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \
-        "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}"
-fi;

taskcluster/pyenv-win-amd64.yml

@@ -1,5 +1,8 @@
 build:
-  template_file: pyenv-win-opt-base.tyml
+  template_file: generic_tc_caching-win-opt-base.tyml
+  cache:
+    artifact_url: "${system.pyenv.win.url}"
+    artifact_namespace: "${system.pyenv.win.namespace}"
   scripts:
     build: "taskcluster/pyenv-build.sh"
     package: "taskcluster/pyenv-package.sh"

taskcluster/pyenv-win-opt-base.tyml

@@ -1,59 +0,0 @@
-$if: 'event.event in build.allowed'
-then:
-  taskId: ${taskcluster.taskId}
-  provisionerId: ${taskcluster.docker.provisionerId}
-  workerType: ${taskcluster.docker.workerTypeWin}
-  taskGroupId: ${taskcluster.taskGroupId}
-  schedulerId: ${taskcluster.schedulerId}
-  created: { $fromNow: '0 sec' }
-  deadline: { $fromNow: '1 day' }
-  expires: { $fromNow: '6 months' }
-  scopes:
-    - "index:insert-task:project.deepspeech.*"
-
-  payload:
-    maxRunTime: { $eval: to_int(build.maxRunTime) }
-
-    features:
-      taskclusterProxy: true
-
-    mounts:
-      - file: msys2-base-x86_64.tar.xz
-        content:
-          sha256: c4443113497acb2d2e285d40b929fc55f33f8f669902595ecdf66a655b63dc60
-          url: >-
-            https://github.com/msys2/msys2-installer/releases/download/2020-05-17/msys2-base-x86_64-20200517.tar.xz
-
-    env:
-      TC_MSYS_VERSION: 'MSYS_NT-6.3-9600'
-      MSYS: 'winsymlinks:nativestrict'
-
-    command:
-      - >-
-        "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
-        "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
-      - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
-      - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
-      - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
-      - $let:
-          taskIndexExpire: { $fromNow: '6 months' }
-        in: >
-          echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C &&
-          export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" &&
-          export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" &&
-          export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
-          echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh &&
-          env && pacman --noconfirm -R bsdtar && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${system.pyenv.win.url}`\" != \"200\" ]; then git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ${taskIndexExpire} taskcluster ${system.pyenv.win.namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k
-
-      - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE"
-
-    artifacts:
-      - type: "directory"
-        path: "public/"
-        expires: { $fromNow: '6 months' }
-
-  metadata:
-    name: ${build.metadata.name}
-    description: ${build.metadata.description}
-    owner: ${event.head.user.email}
-    source: ${event.head.repo.url}

taskcluster/swig-darwin-amd64.yml

@@ -1,7 +1,12 @@
 build:
-  template_file: swig-darwin-opt-base.tyml
+  template_file: generic_tc_caching-darwin-opt-base.tyml
-  swig_system: "darwin"
+  build_or_cache:
-  swig_arch: "amd64"
+    repo: "${system.swig.repo}"
+    sha: "${system.swig.sha1}"
+    dir: "swig"
+  cache:
+    artifact_url: "${system.swig_build.osx.url}"
+    artifact_namespace: "${system.swig_build.osx.namespace}"
   scripts:
     build: "taskcluster/build.sh"
     package: "taskcluster/package.sh"

taskcluster/swig-darwin-opt-base.tyml

@@ -1,56 +0,0 @@
-taskId: ${taskcluster.taskId}
-provisionerId: ${taskcluster.generic.provisionerId}
-workerType: ${taskcluster.generic.workerType}
-taskGroupId: ${taskcluster.taskGroupId}
-schedulerId: ${taskcluster.schedulerId}
-dependencies:
-  $map: { $eval: build.dependencies }
-  each(b):
-    $eval: as_slugid(b)
-created: { $fromNow: '0 sec' }
-deadline: { $fromNow: '1 day' }
-expires: { $fromNow: '6 months' }
-scopes:
-  - "index:insert-task:project.deepspeech.*"
-
-payload:
-  maxRunTime: { $eval: to_int(build.maxRunTime) }
-
-  command:
-    - - "/bin/bash"
-      - "--login"
-      - "-cxe"
-      - $let:
-          taskIndexExpire: { $fromNow: '6 months' }
-        in: >
-          export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
-          export TASKCLUSTER_TASKDIR="$(pwd)" &&
-          export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
-          export LC_ALL=C &&
-          export MACOSX_DEPLOYMENT_TARGET=10.10 &&
-          export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
-          export HOMEBREW_NO_AUTO_UPDATE=1 &&
-          env &&
-          mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
-          swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` &&
-          if [ "$swig_bin" != "200" ]; then
-            git clone --quiet ${system.swig.repo} $TASKCLUSTER_TASKDIR/swig/ &&
-            cd $TASKCLUSTER_TASKDIR/swig/ && git checkout --quiet ${system.swig.sha1} &&
-            $TASKCLUSTER_TASKDIR/swig/${build.scripts.build} &&
-            $TASKCLUSTER_TASKDIR/swig/${build.scripts.package} &&
-            curl -sSL --fail -X PUT \
-                -H "Content-Type: application/json" \
-                -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
-                "http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
-          fi;
-
-  artifacts:
-    - type: "directory"
-      path: "public/"
-      expires: { $fromNow: '6 months' }
-
-metadata:
-  name: ${build.metadata.name}
-  description: ${build.metadata.description}
-  owner: ${event.head.user.email}
-  source: ${event.head.repo.url}

taskcluster/swig-linux-amd64.yml

@@ -1,8 +1,13 @@
 build:
-  template_file: swig-linux-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   docker_image: "ubuntu:14.04"
-  swig_system: "linux"
+  build_or_cache:
-  swig_arch: "amd64"
+    repo: "${system.swig.repo}"
+    sha: "${system.swig.sha1}"
+    dir: "swig"
+  cache:
+    artifact_url: "${system.swig_build.linux.url}"
+    artifact_namespace: "${system.swig_build.linux.namespace}"
   system_setup:
     >
       apt-get -qq -y install autoconf automake bison build-essential

taskcluster/swig-linux-opt-base.tyml

@@ -1,54 +0,0 @@
-$if: 'event.event in build.allowed'
-then:
-  taskId: ${taskcluster.taskId}
-  provisionerId: ${taskcluster.docker.provisionerId}
-  workerType: ${taskcluster.docker.workerType}
-  taskGroupId: ${taskcluster.taskGroupId}
-  schedulerId: ${taskcluster.schedulerId}
-  created: { $fromNow: '0 sec' }
-  deadline: { $fromNow: '1 day' }
-  expires: { $fromNow: '6 months' }
-  scopes:
-    - "index:insert-task:project.deepspeech.*"
-
-  payload:
-    maxRunTime: { $eval: to_int(build.maxRunTime) }
-    image: ${build.docker_image}
-
-    features:
-      taskclusterProxy: true
-
-    command:
-      - "/bin/bash"
-      - "--login"
-      - "-cxe"
-      - $let:
-          extraSystemSetup: { $eval: strip(str(build.system_setup)) }
-          taskIndexExpire: { $fromNow: '6 months' }
-        in: >
-          (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
-          apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
-          swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` &&
-          if [ "$swig_bin" != "200" ]; then
-            adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ &&
-            echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${system.swig.repo} ~/swig/ && cd ~/swig/ && git checkout --quiet ${system.swig.sha1})" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
-            sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
-            sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/swig/${build.scripts.build} &&
-            sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/swig/${build.scripts.package} &&
-            curl -sSL --fail -X PUT \
-                -H "Content-Type: application/json" \
-                -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
-                "http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
-          fi;
-
-    artifacts:
-      "public":
-        type: "directory"
-        path: "/tmp/artifacts/"
-        expires: { $fromNow: '6 months' }
-
-  metadata:
-    name: ${build.metadata.name}
-    description: ${build.metadata.description}
-    owner: ${event.head.user.email}
-    source: ${event.head.repo.url}

taskcluster/swig-win-amd64.yml

@@ -1,8 +1,13 @@
 build:
-  template_file: swig-linux-opt-base.tyml
+  template_file: generic_tc_caching-linux-opt-base.tyml
   docker_image: "ubuntu:18.04"
-  swig_system: "win"
+  build_or_cache:
-  swig_arch: "amd64"
+    repo: "${system.swig.repo}"
+    sha: "${system.swig.sha1}"
+    dir: "swig"
+  cache:
+    artifact_url: "${system.swig_build.win.url}"
+    artifact_namespace: "${system.swig_build.win.namespace}"
   system_setup:
     >
       apt-get -qq -y install autoconf automake bison build-essential mingw-w64 &&

taskcluster/tc-update-index.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Helper script because it is way too painful to deal with Windows' CMD.exe
+# ways of escaping things when pushing JSON
+
+set -xe
+
+TC_EXPIRE=$1
+TC_INSTANCE=$2
+TC_INDEX=$3
+
+source $(dirname "$0")/tc-tests-utils.sh
+
+if [ ! -z "${TC_EXPIRE}" -a ! -z "${TC_INSTANCE}" -a ! -z "${TC_INDEX}" ]; then
+    curl -sSL --fail -X PUT \
+        -H "Content-Type: application/json" \
+        -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${TC_EXPIRE}\",\"data\":{}}" \
+        "http://${TC_INSTANCE}/index/v1/task/${TC_INDEX}"
+fi;