Merge pull request #2705 from lissyx/nodejs-tflite

Produce TFLite-specific NPM package
2020-01-31 11:48:35 +01:00 · 2020-01-31 11:48:35 +01:00 · 8c5a7b07c9
parent 5d0e4cc8ed 6e521ff3a2
commit 8c5a7b07c9
9 changed files with 104 additions and 11 deletions
--- a/taskcluster/node-build.sh
+++ b/taskcluster/node-build.sh
@ -2,10 +2,10 @@

 set -xe

-cuda=$1
+package_option=$1

 source $(dirname "$0")/tc-tests-utils.sh

 source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh

-do_deepspeech_npm_package "${cuda}"
+do_deepspeech_npm_package "${package_option}"
--- a/taskcluster/node-package-tflite.yml
+++ b/taskcluster/node-package-tflite.yml
@ -0,0 +1,24 @@
+build:
+  template_file: linux-opt-base.tyml
+  dependencies:
+    - "darwin-amd64-tflite-opt"
+    - "linux-amd64-tflite-opt"
+    - "win-amd64-tflite-opt"
+  routes:
+    - "notify.irc-channel.${notifications.irc}.on-exception"
+    - "notify.irc-channel.${notifications.irc}.on-failed"
+  system_setup:
+    >
+      ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning}
+      && apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
+      apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script}
+  system_config:
+    >
+      ${swig.patch_nodejs.linux}
+  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz"
+  scripts:
+    build: "taskcluster/node-build.sh --tflite"
+    package: "taskcluster/node-package.sh"
+  metadata:
+    name: "DeepSpeech NodeJS TFLite package"
+    description: "Packaging DeepSpeech TFLite for registry"
--- a/taskcluster/scriptworker-task-github.yml
+++ b/taskcluster/scriptworker-task-github.yml
@ -11,6 +11,7 @@ build:
    - "linux-rpi3-cpu-opt"
    - "node-package-gpu"
    - "node-package-cpu"
+    - "node-package-tflite"
    - "android-arm64-cpu-opt"
    - "android-armv7-cpu-opt"
    - "android-java-opt"
@ -41,6 +42,8 @@ build:
      - "node-package-gpu"
      # CPU package with all archs
      - "node-package-cpu"
+      # tflite package for non-default tflite archs
+      - "node-package-tflite"
    cpp:
      - "darwin-amd64-cpu-opt"
      - "linux-amd64-cpu-opt"
--- a/taskcluster/scriptworker-task-npm.yml
+++ b/taskcluster/scriptworker-task-npm.yml
@ -4,6 +4,7 @@ build:
    # Make sure builds are ready
    - "node-package-gpu"
    - "node-package-cpu"
+    - "node-package-tflite"
  allowed:
    - "tag"
  ref_match: "refs/tags/"
@ -21,6 +22,8 @@ build:
      - "node-package-gpu"
      # CPU package with all archs
      - "node-package-cpu"
+      # tflite package for non-default tflite archs
+      - "node-package-tflite"
    nuget: []
  metadata:
    name: "DeepSpeech NPM Packages"
--- a/taskcluster/tc-tests-utils.sh
+++ b/taskcluster/tc-tests-utils.sh
@ -1195,8 +1195,10 @@ get_python_pkg_url()
 get_dep_npm_pkg_url()
 {
  local all_deps="$(curl -s https://community-tc.services.mozilla.com/api/queue/v1/task/${TASK_ID} | python -c 'import json; import sys; print(" ".join(json.loads(sys.stdin.read())["dependencies"]));')"
-  local deepspeech_pkg="deepspeech-${DS_VERSION}.tgz"

+  # We try "deepspeech-tflite" first and if we don't find it we try "deepspeech"
+  for pkg_basename in "deepspeech-tflite" "deepspeech"; do
+    local deepspeech_pkg="${pkg_basename}-${DS_VERSION}.tgz"
    for dep in ${all_deps}; do
      local has_artifact=$(curl -s https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts | python -c 'import json; import sys; has_artifact = True in [ e["name"].find("'${deepspeech_pkg}'") > 0 for e in json.loads(sys.stdin.read())["artifacts"] ]; print(has_artifact)')
      if [ "${has_artifact}" = "True" ]; then
@ -1204,6 +1206,7 @@ get_dep_npm_pkg_url()
        exit 0
      fi;
    done;
+  done;

  echo ""
  # This should not be reached, otherwise it means we could not find a matching nodejs package
@ -1457,7 +1460,7 @@ do_deepspeech_nodejs_build()

 do_deepspeech_npm_package()
 {
-  rename_to_gpu=$1
+  package_option=$1

  cd ${DS_DSDIR}

@ -1483,8 +1486,10 @@ do_deepspeech_npm_package()
    curl -L https://community-tc.services.mozilla.com/api/queue/v1/task/${dep}/artifacts/public/wrapper.tar.gz | tar -C native_client/javascript -xzvf -
  done;

-  if [ "${rename_to_gpu}" = "--cuda" ]; then
+  if [ "${package_option}" = "--cuda" ]; then
    make -C native_client/javascript clean npm-pack PROJECT_NAME=deepspeech-gpu
+  elif [ "${package_option}" = "--tflite" ]; then
+    make -C native_client/javascript clean npm-pack PROJECT_NAME=deepspeech-tflite
  else
    make -C native_client/javascript clean npm-pack
  fi
--- a/taskcluster/test-nodejs_13x_16k_multiarchpkg-linux-tflite-opt.yml
+++ b/taskcluster/test-nodejs_13x_16k_multiarchpkg-linux-tflite-opt.yml
@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  docker_image: "ubuntu:16.04"
+  dependencies:
+    - "node-package-tflite"
+    - "test-training_16k-linux-amd64-py36m-opt"
+  test_model_task: "test-training_16k-linux-amd64-py36m-opt"
+  system_setup:
+    >
+      ${nodejs.packages_xenial.prep_13} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k"
+  metadata:
+    name: "DeepSpeech Linux AMD64 TFLite NodeJS MultiArch Package 13.x tests (16kHz)"
+    description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v13.x, TFLite only, optimized version (16kHz)"
--- a/taskcluster/test-nodejs_13x_8k_multiarchpkg-linux-tflite-opt.yml
+++ b/taskcluster/test-nodejs_13x_8k_multiarchpkg-linux-tflite-opt.yml
@ -0,0 +1,15 @@
+build:
+  template_file: test-linux-opt-base.tyml
+  docker_image: "ubuntu:16.04"
+  dependencies:
+    - "node-package-tflite"
+    - "test-training_8k-linux-amd64-py36m-opt"
+  test_model_task: "test-training_8k-linux-amd64-py36m-opt"
+  system_setup:
+    >
+      ${nodejs.packages_xenial.prep_13} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
+  args:
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 8k"
+  metadata:
+    name: "DeepSpeech Linux AMD64 TFLite NodeJS MultiArch Package 13.x tests (8kHz)"
+    description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v13.x, TFLite only, optimized version (8kHz)"
--- a/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml
+++ b/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml
@ -0,0 +1,14 @@
+build:
+  template_file: test-darwin-opt-base.tyml
+  dependencies:
+    - "node-package-tflite"
+    - "test-training_16k-linux-amd64-py36m-opt"
+  test_model_task: "test-training_16k-linux-amd64-py36m-opt"
+  system_setup:
+    >
+      ${nodejs.brew.setup} && ${nodejs.brew.prep_13} && ${nodejs.brew.env}
+  args:
+    tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k"
+  metadata:
+    name: "DeepSpeech OSX AMD64 TFLite NodeJS MultiArch Package 13.x tests"
+    description: "Testing DeepSpeech for OSX/AMD64 on NodeJS MultiArch Package v13.x, TFLite only, optimized version"
--- a/taskcluster/test-nodejs_13x_multiarchpkg-win-tflite-opt.yml
+++ b/taskcluster/test-nodejs_13x_multiarchpkg-win-tflite-opt.yml
@ -0,0 +1,14 @@
+build:
+  template_file: test-win-opt-base.tyml
+  dependencies:
+    - "node-package-tflite"
+    - "test-training_16k-linux-amd64-py36m-opt"
+  test_model_task: "test-training_16k-linux-amd64-py36m-opt"
+  system_setup:
+    >
+      ${system.sox_win} && ${nodejs.win.prep_13}
+  args:
+    tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k"
+  metadata:
+    name: "DeepSpeech Windows AMD64 TFLite NodeJS MultiArch Package 13.x tests"
+    description: "Testing DeepSpeech for Windows/AMD64 on NodeJS MultiArch Package v13.x, TFLite only, optimized version"