All NodeJS into one package

2017-11-20 09:59:40 +01:00 · 2017-11-20 09:59:40 +01:00 · 964e38fccd
commit 964e38fccd
parent e56bd20e37
17 changed files with 81 additions and 18 deletions
--- a/native_client/definitions.mk
+++ b/native_client/definitions.mk
@ -46,12 +46,12 @@ LD      := $(TOOLCHAIN)ld
 LDD     := $(TOOLCHAIN)ldd $(TOOLCHAIN_LDD_OPTS)

 RPATH_PYTHON         := '-Wl,-rpath,\$$ORIGIN/lib/'
-RPATH_NODEJS         := '-Wl,-rpath,$$\$$ORIGIN/../../../'
+RPATH_NODEJS         := '-Wl,-rpath,$$\$$ORIGIN/../'
 META_LD_LIBRARY_PATH := LD_LIBRARY_PATH
 ifeq ($(OS),Darwin)
 META_LD_LIBRARY_PATH := DYLD_LIBRARY_PATH
 RPATH_PYTHON         := '-Wl,-rpath,@loader_path/lib/'
-RPATH_NODEJS         := '-Wl,-rpath,@loader_path/../../../'
+RPATH_NODEJS         := '-Wl,-rpath,@loader_path/../'
 endif

 # Takes care of looking into bindings built (SRC_FILE, can contain a wildcard)
--- a/native_client/javascript/Makefile
+++ b/native_client/javascript/Makefile
@ -24,9 +24,9 @@ build: configure deepspeech_wrap.cxx
 	AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(RPATH_NODEJS) $(LDFLAGS)" LIBS="$(LIBS)" $(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) rebuild $(NODE_BUILD_VERBOSE)

 copy-deps: build
-	$(call copy_missing_libs,lib/binding/*/*/deepspeech.node,lib/)
+	$(call copy_missing_libs,lib/binding/*/*/*/deepspeech.node,lib/binding/*/*/)

-package: copy-deps build
+node-wrapper: copy-deps build
 	$(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) package $(NODE_BUILD_VERBOSE)

 npm-pack: clean
--- a/native_client/javascript/package.json
+++ b/native_client/javascript/package.json
@ -10,7 +10,7 @@
    },
    "binary": {
        "module_name" : "deepspeech",
-        "module_path" : "./lib/binding/v{version}/{node_abi}-{platform}-{arch}/",
+        "module_path" : "./lib/binding/v{version}/{platform}-{arch}/{node_abi}/",
        "remote_path" : "./v{version}/{configuration}/",
        "package_name": "{module_name}-v{version}-{node_abi}-{platform}-{arch}.tar.gz",
        "host"        : "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.v1.0.0-warpctc.arm/artifacts/public/"
--- a/taskcluster/node-build.sh
+++ b/taskcluster/node-build.sh
@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/../tc-tests-utils.sh
+
+source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+
+do_deepspeech_npm_package
--- a/taskcluster/node-package.sh
+++ b/taskcluster/node-package.sh
@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname "$0")/../tc-tests-utils.sh
+
+mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
+
+# NodeJS package
+cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/deepspeech-*.tgz ${TASKCLUSTER_ARTIFACTS}/
--- a/taskcluster/node-package.yml
+++ b/taskcluster/node-package.yml
@ -0,0 +1,21 @@
+build:
+  template_file: linux-opt-base.tyml
+  dependencies:
+    - "darwin-amd64-cpu-opt"
+    - "linux-amd64-cpu-opt"
+    - "linux-rpi3-cpu-opt"
+  routes:
+    - "notify.irc-channel.${notifications.irc}.on-exception"
+    - "notify.irc-channel.${notifications.irc}.on-failed"
+  system_setup:
+    >
+      ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml &&
+      apt-get -qq -y install ${python.packages.apt} && ${swig.packages.install_script}
+  tensorflow: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/home.tar.xz"
+  summarize_graph: "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/summarize_graph"
+  scripts:
+    build: "taskcluster/node-build.sh"
+    package: "taskcluster/node-package.sh"
+  metadata:
+    name: "DeepSpeech NodeJS package"
+    description: "Packaging DeepSpeech for registry"
--- a/taskcluster/package.sh
+++ b/taskcluster/package.sh
@ -9,9 +9,10 @@ mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
 package_native_client "native_client.tar.xz"

 if [ -d ${DS_ROOT_TASK}/DeepSpeech/ds/wheels ]; then
-    # Python wheels
    cp ${DS_ROOT_TASK}/DeepSpeech/ds/wheels/* ${TASKCLUSTER_ARTIFACTS}/
-
-    # NodeJS package
    cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/deepspeech-*.tgz ${TASKCLUSTER_ARTIFACTS}/
 fi;
+
+if [ -f ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/wrapper.tar.gz ]; then
+    cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/javascript/wrapper.tar.gz ${TASKCLUSTER_ARTIFACTS}/
+fi;
--- a/taskcluster/test-linux-opt-base.tyml
+++ b/taskcluster/test-linux-opt-base.tyml
@ -35,9 +35,11 @@ then:
        linux_amd64_aot_test: { $eval: as_slugid("linux-amd64-cpu-aot_test-opt") }
        linux_amd64_build: { $eval: as_slugid("linux-amd64-cpu-opt") }
        linux_amd64_ctc: { $eval: as_slugid("linux-amd64-ctc-opt") }
+        node_package: { $eval: as_slugid("node-package") }
      in:
        TENSORFLOW_WHEEL: https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.master.cpu/artifacts/public/tensorflow_warpctc-1.3.0rc0-cp27-cp27mu-linux_x86_64.whl
        DEEPSPEECH_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_build}/artifacts/public
+        DEEPSPEECH_NODEJS: https://queue.taskcluster.net/v1/task/${node_package}/artifacts/public
        DEEPSPEECH_AOT_ARTIFACTS_ROOT: https://queue.taskcluster.net/v1/task/${linux_amd64_aot_test}/artifacts/public
        DEEPSPEECH_LIBCTC: https://queue.taskcluster.net/v1/task/${linux_amd64_ctc}/artifacts/public/decoder.tar.xz
        DEEPSPEECH_TEST_MODEL: https://queue.taskcluster.net/v1/task/${training}/artifacts/public/output_graph.pb
--- a/taskcluster/test-nodejs_4x-linux-amd64-opt.yml
+++ b/taskcluster/test-nodejs_4x-linux-amd64-opt.yml
@ -1,7 +1,7 @@
 build:
  template_file: test-linux-opt-base.tyml
  dependencies:
-    - "linux-amd64-cpu-opt"
+    - "node-package"
    - "test-training-linux-amd64-opt"
  system_setup:
    >
--- a/taskcluster/test-nodejs_4x-linux-amd64-prod_pbmodel-opt.yml
+++ b/taskcluster/test-nodejs_4x-linux-amd64-prod_pbmodel-opt.yml
@ -1,7 +1,7 @@
 build:
  template_file: test-linux-opt-base.tyml
  dependencies:
-    - "linux-amd64-cpu-opt"
+    - "node-package"
    - "test-training-linux-amd64-opt"
  system_setup:
    >
--- a/taskcluster/test-nodejs_5x-linux-amd64-opt.yml
+++ b/taskcluster/test-nodejs_5x-linux-amd64-opt.yml
@ -1,7 +1,7 @@
 build:
  template_file: test-linux-opt-base.tyml
  dependencies:
-    - "linux-amd64-cpu-opt"
+    - "node-package"
    - "test-training-linux-amd64-opt"
  system_setup:
    >
--- a/taskcluster/test-nodejs_5x-linux-amd64-prod_pbmodel-opt.yml
+++ b/taskcluster/test-nodejs_5x-linux-amd64-prod_pbmodel-opt.yml
@ -1,7 +1,7 @@
 build:
  template_file: test-linux-opt-base.tyml
  dependencies:
-    - "linux-amd64-cpu-opt"
+    - "node-package"
    - "test-training-linux-amd64-opt"
  system_setup:
    >
--- a/taskcluster/test-nodejs_6x-linux-amd64-opt.yml
+++ b/taskcluster/test-nodejs_6x-linux-amd64-opt.yml
@ -1,7 +1,7 @@
 build:
  template_file: test-linux-opt-base.tyml
  dependencies:
-    - "linux-amd64-cpu-opt"
+    - "node-package"
    - "test-training-linux-amd64-opt"
  system_setup:
    >
--- a/taskcluster/test-nodejs_6x-linux-amd64-prod_pbmodel-opt.yml
+++ b/taskcluster/test-nodejs_6x-linux-amd64-prod_pbmodel-opt.yml
@ -1,13 +1,13 @@
 build:
  template_file: test-linux-opt-base.tyml
  dependencies:
-    - "linux-amd64-cpu-opt"
+    - "node-package"
    - "test-training-linux-amd64-opt"
  system_setup:
    >
      ${nodejs.packages.prep_6} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages.apt}
  args:
-          tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 6.x"
+    tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 6.x"
  metadata:
    name: "DeepSpeech Linux AMD64 CPU NodeJS 6.x prod tests"
    description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v6.x on prod model, CPU only, optimized version"
--- a/tc-node-tests-prod.sh
+++ b/tc-node-tests-prod.sh
@ -19,7 +19,7 @@ download_data
 pushd ${HOME}/DeepSpeech/ds/native_client/
    node --version
    npm --version
-    npm install ${DEEPSPEECH_ARTIFACTS_ROOT}/deepspeech-0.0.2.tgz
+    npm install ${DEEPSPEECH_NODEJS}/deepspeech-0.0.2.tgz
    npm install

    phrase_pbmodel_withlm=$(node client.js /tmp/${model_name} /tmp/LDC93S1.wav /tmp/alphabet.txt /tmp/lm.binary /tmp/trie)
--- a/tc-node-tests.sh
+++ b/tc-node-tests.sh
@ -20,7 +20,7 @@ pushd ${HOME}/DeepSpeech/ds/native_client/
    if [ "${aot_model}" = "--aot" ]; then
        npm install ${DEEPSPEECH_AOT_ARTIFACTS_ROOT}/deepspeech-0.0.2.tgz
    else
-        npm install ${DEEPSPEECH_ARTIFACTS_ROOT}/deepspeech-0.0.2.tgz
+        npm install ${DEEPSPEECH_NODEJS}/deepspeech-0.0.2.tgz
    fi

    npm install
--- a/tc-tests-utils.sh
+++ b/tc-tests-utils.sh
@ -297,7 +297,7 @@ do_deepspeech_nodejs_build()
      RASPBIAN=/tmp/multistrap-raspbian-jessie \
      TFDIR=${DS_TFDIR} \
      NODE_ABI_TARGET=--target=$node \
-      clean package
+      clean node-wrapper
  done;

  make -C native_client/javascript clean npm-pack
@ -306,6 +306,26 @@ do_deepspeech_nodejs_build()
    pkg=(native_client/javascript/deepspeech-*.tgz)
    mv "${pkg}" "${pkg//deepspeech/deepspeech-gpu}"
  fi
+
+  tar -czf native_client/javascript/wrapper.tar.gz \
+    -C native_client/javascript/ lib/
+}
+
+do_deepspeech_npm_package()
+{
+  cd ${DS_DSDIR}
+
+  npm update && npm install node-gyp node-pre-gyp
+
+  export PATH="$(npm root)/.bin/:$PATH"
+
+  all_tasks="$(curl -s https://queue.taskcluster.net/v1/task/${TASK_ID} | python -c 'import json; import sys; print(" ".join(json.loads(sys.stdin.read())["dependencies"]));')"
+
+  for dep in ${all_tasks}; do
+    curl -L https://queue.taskcluster.net/v1/task/${dep}/artifacts/public/wrapper.tar.gz | tar -C native_client/javascript -xzvf -
+  done;
+
+  make -C native_client/javascript clean npm-pack
 }

 package_native_client()