Manually update the TC index when needed

.taskcluster.yml

@@ -27,6 +27,7 @@ tasks:
     scopes: [
       "queue:create-task:highest:proj-deepspeech/*",
       "queue:route:index.project.deepspeech.*",
+      "index:insert-task:project.deepspeech.*",
       "queue:route:notify.irc-channel.*",
       "queue:scheduler-id:taskcluster-github",
       "generic-worker:cache:deepspeech-homebrew-bin",

taskcluster/.shared.yml

@@ -59,14 +59,16 @@ cpp:
     packages: 'install_pkg_local_homebrew "sox"'
     env: 'export EXTRA_ENV="DYLD_LIBRARY_PATH=$LOCAL_HOMEBREW_DIRECTORY/lib/:$DYLD_LIBRARY_PATH"'
 system:
-  node_gyp_cache: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.0/artifacts/public/node-gyp-cache.tar.gz'
+  node_gyp_cache:
+    url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.1/artifacts/public/node-gyp-cache.tar.gz'
+    namespace: 'project.deepspeech.node-gyp-cache.1'
   swig:
     repo: "https://github.com/lissyx/swig"
-    sha1: "04baa423c41008712e811cd5c1ee94e9e90894aa"
+    sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
     cache:
-      linux_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.04baa423c41008712e811cd5c1ee94e9e90894aa/artifacts/public/ds-swig.tar.gz'
+      linux_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.linux.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
-      darwin_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.04baa423c41008712e811cd5c1ee94e9e90894aa/artifacts/public/ds-swig.tar.gz'
+      darwin_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.darwin.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
-      win_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.04baa423c41008712e811cd5c1ee94e9e90894aa/artifacts/public/ds-swig.tar.gz'
+      win_amd64: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz'
   username: 'build-user'
   homedir:
     linux: '/home/build-user'

taskcluster/android-arm64-cpu-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64"

taskcluster/android-armv7-cpu-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7"

taskcluster/android-x86_64-cpu-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64"

taskcluster/darwin-amd64-cpu-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: darwin-opt-base.tyml
   dependencies:
     - "swig-darwin-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx"

taskcluster/darwin-amd64-ctc-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: darwin-opt-base.tyml
   dependencies:
     - "swig-darwin-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc"

taskcluster/darwin-amd64-tflite-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: darwin-opt-base.tyml
   dependencies:
     - "swig-darwin-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite"

taskcluster/darwin-opt-base.tyml

@@ -125,7 +125,7 @@ payload:
       - directory: ${system.homedir.osx}/Library/Caches/node-gyp/
         format: tar.gz
         content:
-          url: ${system.node_gyp_cache}
+          url: ${system.node_gyp_cache.url}
       - directory: ds-swig
         format: tar.gz
         content:
@@ -134,7 +134,7 @@ payload:
       - directory: ${system.homedir.osx}/Library/Caches/node-gyp/
         format: tar.gz
         content:
-          url: ${system.node_gyp_cache}
+          url: ${system.node_gyp_cache.url}
       - directory: ds-swig
         format: tar.gz
         content:

taskcluster/linux-amd64-cpu-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu"

taskcluster/linux-amd64-ctc-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-ctc"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-ctc"

taskcluster/linux-amd64-gpu-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu"

taskcluster/linux-amd64-tflite-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite"

taskcluster/linux-arm64-cpu-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"

taskcluster/linux-opt-base.tyml

@@ -48,7 +48,7 @@ then:
           adduser --system --home ${system.homedir.linux} ${system.username} &&
           apt-get -qq update && apt-get -qq -y install git pixz wget pkg-config libsox-dev && ${extraSystemSetup} &&
           cd ${system.homedir.linux}/ &&
-          echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/ds-swig/bin/ && wget -O - ${system.swig.cache.linux_amd64} | tar -C ${system.homedir.linux}/ds-swig/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
           sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
           ${system.homedir.linux}/DeepSpeech/tf/tc-apt.sh && ${extraSystemConfig} &&
           sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&

taskcluster/linux-rpi3-cpu-opt.yml

@@ -2,6 +2,7 @@ build:
   template_file: linux-opt-base.tyml
   dependencies:
     - "swig-linux-amd64"
+    - "node-gyp-cache"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"

taskcluster/node-gyp-cache-base.tyml

@@ -8,12 +8,16 @@ then:
   created: { $fromNow: '0 sec' }
   deadline: { $fromNow: '1 day' }
   expires: { $fromNow: '6 months' }
-  routes: { $eval: build.routes }
+  scopes:
+    - "index:insert-task:project.deepspeech.*"
 
   payload:
     maxRunTime: { $eval: to_int(build.maxRunTime) }
     image: "node:12"
 
+    features:
+      taskclusterProxy: true
+
     # This task will inspect system.node_gyp_cache taskcluster index existence:
     #  - if the artifact does not exists, it will build it
     #  - if the artifact exists, it will re-mirror it (if we don't do that, new
@@ -24,18 +28,20 @@ then:
       - "-cxe"
       - $let:
           extraSystemSetup: { $eval: strip(str(build.system_setup)) }
-          extraSystemConfig: { $eval: strip(str(build.system_config)) }
+          taskIndexExpire: { $fromNow: '6 months' }
         in: >
           apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
-          cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache}` &&
+          cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache.url}` &&
           if [ "$cache_file" != "200" ]; then
             mkdir -p ~/DeepSpeech/ds/ &&
             git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ &&
             cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
             ~/DeepSpeech/ds/${build.scripts.build} &&
-            ~/DeepSpeech/ds/${build.scripts.package}
+            ~/DeepSpeech/ds/${build.scripts.package} &&
-          else
+            curl -sSL --fail -X PUT \
-            mkdir -p /tmp/artifacts/ && curl -sSL -o /tmp/artifacts/node-gyp-cache.tar.gz ${system.node_gyp_cache}
+                -H "Content-Type: application/json" \
+                -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
+                "http://taskcluster/index/v1/task/${system.node_gyp_cache.namespace}"
           fi;
 
     artifacts:

taskcluster/node-gyp-cache.yml

@@ -1,7 +1,5 @@
 build:
   template_file: node-gyp-cache-base.tyml
-  routes:
-    - "index.project.deepspeech.node-gyp-cache.0"
   scripts:
     build: "taskcluster/node-gyp-populate.sh"
     package: "taskcluster/node-gyp-package.sh"

taskcluster/swig-darwin-opt-base.tyml

@@ -10,8 +10,8 @@ dependencies:
 created: { $fromNow: '0 sec' }
 deadline: { $fromNow: '1 day' }
 expires: { $fromNow: '6 months' }
-routes:
+scopes:
-  - "index.project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
+  - "index:insert-task:project.deepspeech.*"
 
 payload:
   maxRunTime: { $eval: to_int(build.maxRunTime) }
@@ -20,24 +20,29 @@ payload:
     - - "/bin/bash"
       - "--login"
       - "-cxe"
-      - >
+      - $let:
-        export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
+          taskIndexExpire: { $fromNow: '6 months' }
-        export TASKCLUSTER_TASKDIR="$(pwd)" &&
+        in: >
-        export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
+          export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
-        export LC_ALL=C &&
+          export TASKCLUSTER_TASKDIR="$(pwd)" &&
-        export MACOSX_DEPLOYMENT_TARGET=10.10 &&
+          export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
-        export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
+          export LC_ALL=C &&
-        export HOMEBREW_NO_AUTO_UPDATE=1 &&
+          export MACOSX_DEPLOYMENT_TARGET=10.10 &&
-        env &&
+          export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
-        swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` &&
+          export HOMEBREW_NO_AUTO_UPDATE=1 &&
-        if [ "$swig_bin" != "200" ]; then
+          env &&
-          git clone --quiet ${system.swig.repo} $TASKCLUSTER_TASKDIR/swig/ &&
+          mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
-          cd $TASKCLUSTER_TASKDIR/swig/ && git checkout --quiet ${system.swig.sha1} &&
+          swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz` &&
-          $TASKCLUSTER_TASKDIR/swig/${build.scripts.build} &&
+          if [ "$swig_bin" != "200" ]; then
-          $TASKCLUSTER_TASKDIR/swig/${build.scripts.package}
+            git clone --quiet ${system.swig.repo} $TASKCLUSTER_TASKDIR/swig/ &&
-        else
+            cd $TASKCLUSTER_TASKDIR/swig/ && git checkout --quiet ${system.swig.sha1} &&
-          mkdir -p $TASKCLUSTER_ARTIFACTS/ && curl -sSL -o $TASKCLUSTER_ARTIFACTS/ds-swig.tar.gz https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz
+            $TASKCLUSTER_TASKDIR/swig/${build.scripts.build} &&
-        fi;
+            $TASKCLUSTER_TASKDIR/swig/${build.scripts.package} &&
+            curl -sSL --fail -X PUT \
+                -H "Content-Type: application/json" \
+                -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
+                "http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
+          fi;
 
   artifacts:
     - type: "directory"

taskcluster/swig-linux-opt-base.tyml

@@ -8,20 +8,23 @@ then:
   created: { $fromNow: '0 sec' }
   deadline: { $fromNow: '1 day' }
   expires: { $fromNow: '6 months' }
-  routes:
+  scopes:
-    - "index.project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
+    - "index:insert-task:project.deepspeech.*"
 
   payload:
     maxRunTime: { $eval: to_int(build.maxRunTime) }
     image: ${build.docker_image}
 
+    features:
+      taskclusterProxy: true
+
     command:
       - "/bin/bash"
       - "--login"
       - "-cxe"
       - $let:
           extraSystemSetup: { $eval: strip(str(build.system_setup)) }
-          extraSystemConfig: { $eval: strip(str(build.system_config)) }
+          taskIndexExpire: { $fromNow: '6 months' }
         in: >
           (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
           apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
@@ -31,9 +34,11 @@ then:
             echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${system.swig.repo} ~/swig/ && cd ~/swig/ && git checkout --quiet ${system.swig.sha1})" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
             sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
             sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/swig/${build.scripts.build} &&
-            sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/swig/${build.scripts.package}
+            sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/swig/${build.scripts.package} &&
-          else
+            curl -sSL --fail -X PUT \
-            mkdir -p /tmp/artifacts/ && curl -sSL -o /tmp/artifacts/ds-swig.tar.gz https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}/artifacts/public/ds-swig.tar.gz
+                -H "Content-Type: application/json" \
+                -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
+                "http://taskcluster/index/v1/task/project.deepspeech.swig.${build.swig_system}.${build.swig_arch}.${system.swig.sha1}"
           fi;
 
     artifacts:

taskcluster/win-opt-base.tyml

@@ -37,7 +37,7 @@ payload:
     - directory: .node-gyp
       format: tar.gz
       content:
-        url: ${system.node_gyp_cache}
+        url: ${system.node_gyp_cache.url}
     - directory: ds-swig
       format: tar.gz
       content: