Merge pull request #2737 from lissyx/win-nodegyp-cache

Produce and use node-gyp cache

taskcluster/.shared.yml

@@ -80,6 +80,7 @@ cpp:
     packages: 'install_pkg_local_homebrew "sox"'
     env: 'export EXTRA_ENV="DYLD_LIBRARY_PATH=$LOCAL_HOMEBREW_DIRECTORY/lib/:$DYLD_LIBRARY_PATH"'
 system:
+  node_gyp_cache: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.0/artifacts/public/node-gyp-cache.tar.gz'
   username: 'build-user'
   homedir:
     linux: '/home/build-user'

taskcluster/darwin-opt-base.tyml

@@ -112,6 +112,8 @@ payload:
         then: { $fromNow: '6 months' }
         else: { $fromNow: '7 days' }
 
+  # Define node gyp cache mount twice because we cannot have it separated
+  # from the if/then without the parser to break
   mounts:
     $if: '(event.event != "push") && (event.event != "tag")'
     then:
@@ -121,6 +123,15 @@ payload:
         directory: homebrew.cache/
       - cacheName: deepspeech-macos-pyenv
         directory: pyenv.cache/
+      - directory: ${system.homedir.osx}/Library/Caches/node-gyp/
+        format: tar.gz
+        content:
+          url: ${system.node_gyp_cache}
+    else:
+      - directory: ${system.homedir.osx}/Library/Caches/node-gyp/
+        format: tar.gz
+        content:
+          url: ${system.node_gyp_cache}
 
 metadata:
   name: ${build.metadata.name}

taskcluster/linux-opt-base.tyml

@@ -48,7 +48,7 @@ then:
           apt-get -qq update && apt-get -qq -y install git pixz wget pkg-config libsox-dev && ${extraSystemSetup} &&
           adduser --system --home ${system.homedir.linux} ${system.username} &&
           cd ${system.homedir.linux}/ &&
-          echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
           sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
           ${system.homedir.linux}/DeepSpeech/tf/tc-apt.sh && ${extraSystemConfig} &&
           sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&

taskcluster/node-gyp-cache-base.tyml

@@ -0,0 +1,51 @@
+$if: 'event.event in build.allowed'
+then:
+  taskId: ${taskcluster.taskId}
+  provisionerId: ${taskcluster.docker.provisionerId}
+  workerType: ${taskcluster.docker.workerType}
+  taskGroupId: ${taskcluster.taskGroupId}
+  schedulerId: ${taskcluster.schedulerId}
+  created: { $fromNow: '0 sec' }
+  deadline: { $fromNow: '1 day' }
+  expires: { $fromNow: '6 months' }
+  routes: { $eval: build.routes }
+
+  payload:
+    maxRunTime: { $eval: to_int(build.maxRunTime) }
+    image: "node:12"
+
+    # This task will inspect system.node_gyp_cache taskcluster index existence:
+    #  - if the artifact does not exists, it will build it
+    #  - if the artifact exists, it will re-mirror it (if we don't do that, new
+    #    index gets published with no artifact and erases existing one)
+    command:
+      - "/bin/bash"
+      - "--login"
+      - "-cxe"
+      - $let:
+          extraSystemSetup: { $eval: strip(str(build.system_setup)) }
+          extraSystemConfig: { $eval: strip(str(build.system_config)) }
+        in: >
+          apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup};
+          cache_file=`curl -sSIL -o /dev/null -w "%{http_code}" ${system.node_gyp_cache}` &&
+          if [ "$cache_file" != "200" ]; then
+            mkdir -p ~/DeepSpeech/ds/ &&
+            git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ &&
+            cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
+            ~/DeepSpeech/ds/${build.scripts.build} &&
+            ~/DeepSpeech/ds/${build.scripts.package}
+          else
+            mkdir -p /tmp/artifacts/ && curl -sSL -o /tmp/artifacts/node-gyp-cache.tar.gz ${system.node_gyp_cache}
+          fi;
+
+    artifacts:
+      "public":
+        type: "directory"
+        path: "/tmp/artifacts/"
+        expires: { $fromNow: '6 months' }
+
+  metadata:
+    name: ${build.metadata.name}
+    description: ${build.metadata.description}
+    owner: ${event.head.user.email}
+    source: ${event.head.repo.url}

taskcluster/node-gyp-cache.yml

@@ -0,0 +1,10 @@
+build:
+  template_file: node-gyp-cache-base.tyml
+  routes:
+    - "index.project.deepspeech.node-gyp-cache.0"
+  scripts:
+    build: "taskcluster/node-gyp-populate.sh"
+    package: "taskcluster/node-gyp-package.sh"
+  metadata:
+    name: "Cache for node-gyp headers"
+    description: "Building cache for node-gyp headers"

taskcluster/node-gyp-package.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/tc-tests-utils.sh
+
+mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
+
+cd $DS_ROOT_TASK/node-gyp-cache/ && tar -czf ${TASKCLUSTER_ARTIFACTS}/node-gyp-cache.tar.gz .

taskcluster/node-gyp-populate.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/tc-tests-utils.sh
+
+node --version
+
+npm --version
+
+npm install -g node-gyp@6.x
+
+devDir=$DS_ROOT_TASK/node-gyp-cache/
+
+mkdir -p ${devDir}
+
+node-gyp list ${devDir}
+
+for node in ${SUPPORTED_NODEJS_VERSIONS}; do
+    node-gyp install --devdir ${devDir} \
+        --target=$node
+    mkdir ${devDir}/${node}/x64/ || true
+    curl -sSL https://nodejs.org/dist/v${node}/win-x64/node.lib -o ${devDir}/${node}/x64/node.lib
+done;
+
+for electron in ${SUPPORTED_ELECTRONJS_VERSIONS}; do
+    node-gyp install --devdir  ${devDir} \
+        --target=$electron \
+        --disturl=https://electronjs.org/headers \
+        --runtime=electron
+    mkdir ${devDir}/${electron}/x64/ || true
+    curl -sSL https://electronjs.org/headers/v${electron}/win-x64/node.lib -o ${devDir}/${electron}/x64/node.lib
+done;

taskcluster/tc-tests-utils.sh

@@ -59,6 +59,10 @@ model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}"
 ldc93s1_sample_filename=''
 
 SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-3.5.8:ucs4 3.6.10:ucs4 3.7.6:ucs4 3.8.1:ucs4}
+
+# When updating NodeJS / ElectronJS supported versions, do not forget to increment
+# deepspeech.node-gyp-cache.<X> in both `system.node_gyp_cache` (taskcluster/.shared.yml)
+# and route index (taskcluster/node-gyp-cache.yml) to ensure the cache is updated
 SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-10.18.1 11.15.0 12.8.1 13.1.0}
 SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-5.0.13 6.0.12 6.1.7 7.0.1 7.1.8}
 

taskcluster/win-amd64-cpu-opt.yml

@@ -6,6 +6,8 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.win.${event.head.sha}"
     - "notify.irc-channel.${notifications.irc}.on-exception"
     - "notify.irc-channel.${notifications.irc}.on-failed"
+  dependencies:
+    - "node-gyp-cache"
   tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz"
   scripts:
     build: "taskcluster/win-build.sh"

taskcluster/win-amd64-gpu-opt.yml

@@ -6,6 +6,8 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.win-cuda.${event.head.sha}"
     - "notify.irc-channel.${notifications.irc}.on-exception"
     - "notify.irc-channel.${notifications.irc}.on-failed"
+  dependencies:
+    - "node-gyp-cache"
   tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win-cuda/artifacts/public/home.tar.xz"
   scripts:
     build: "taskcluster/win-build.sh --cuda"

taskcluster/win-amd64-tflite-opt.yml

@@ -6,6 +6,8 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.win-tflite.${event.head.sha}"
     - "notify.irc-channel.${notifications.irc}.on-exception"
     - "notify.irc-channel.${notifications.irc}.on-failed"
+  dependencies:
+    - "node-gyp-cache"
   tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz"
   scripts:
     build: "taskcluster/win-build.sh --tflite"

taskcluster/win-opt-base.tyml

@@ -34,6 +34,10 @@ payload:
         sha256: 4e799b5c3efcf9efcb84923656b7bcff16f75a666911abd6620ea8e5e1e9870c
         url: >-
           https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-base-x86_64-20180531.tar.xz/download
+    - directory: .node-gyp
+      format: tar.gz
+      content:
+        url: ${system.node_gyp_cache}
 
   env:
     TC_MSYS_VERSION: 'MSYS_NT-6.3'