From 5536e5440ee3cf7ea133c003b09f6a3ee8592d71 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Fri, 26 Oct 2018 15:08:31 +0200 Subject: [PATCH] Introduce NodeJS v11.x --- native_client/javascript/Makefile | 2 +- .../javascript/node-pre-gyp_nodejs_v11.patch | 66 +++++++++++++++++++ native_client/javascript/package.json.in | 3 +- taskcluster/.build.yml | 1 + taskcluster/.shared.yml | 8 ++- taskcluster/linux-amd64-cpu-opt.yml | 2 +- taskcluster/linux-amd64-gpu-opt.yml | 2 +- taskcluster/linux-arm64-cpu-opt.yml | 2 +- taskcluster/linux-rpi3-cpu-opt.yml | 2 +- taskcluster/test-linux-opt-base.tyml | 4 +- .../test-nodejs_10x-linux-amd64-opt.yml | 3 +- ...odejs_10x-linux-amd64-prod_pbmodel-opt.yml | 3 +- .../test-nodejs_11x-armbian-arm64-opt.yml | 13 ++++ .../test-nodejs_11x-darwin-amd64-opt.yml | 13 ++++ .../test-nodejs_11x-linux-amd64-opt.yml | 14 ++++ ...odejs_11x-linux-amd64-prod_pbmodel-opt.yml | 13 ++++ .../test-nodejs_11x-raspbian-rpi3-opt.yml | 13 ++++ tc-tests-utils.sh | 11 +++- 18 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 native_client/javascript/node-pre-gyp_nodejs_v11.patch create mode 100644 taskcluster/test-nodejs_11x-armbian-arm64-opt.yml create mode 100644 taskcluster/test-nodejs_11x-darwin-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_11x-linux-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_11x-linux-amd64-prod_pbmodel-opt.yml create mode 100644 taskcluster/test-nodejs_11x-raspbian-rpi3-opt.yml diff --git a/native_client/javascript/Makefile b/native_client/javascript/Makefile index b8a92029..ab8d7519 100644 --- a/native_client/javascript/Makefile +++ b/native_client/javascript/Makefile @@ -41,7 +41,7 @@ node-wrapper: copy-deps build $(NODE_BUILD_TOOL) $(NODE_PLATFORM_TARGET) $(NODE_ABI_TARGET) package $(NODE_BUILD_VERBOSE) npm-pack: clean package.json README.md index.js - npm install node-pre-gyp@0.6.x + npm install node-pre-gyp@0.11.x npm pack $(NODE_BUILD_VERBOSE) deepspeech_wrap.cxx: deepspeech.i diff --git a/native_client/javascript/node-pre-gyp_nodejs_v11.patch b/native_client/javascript/node-pre-gyp_nodejs_v11.patch new file mode 100644 index 00000000..e384227d --- /dev/null +++ b/native_client/javascript/node-pre-gyp_nodejs_v11.patch @@ -0,0 +1,66 @@ +From f356b37c438c726516653ca2fd830359654c5b70 Mon Sep 17 00:00:00 2001 +From: "Nicolas \"Pixel\" Noble" +Date: Sat, 27 Oct 2018 00:25:50 +0200 +Subject: [PATCH] Updating crosswalk information. + +--- + lib/util/abi_crosswalk.json | 32 ++++++++++++++++++++++++++++++++ + 1 file changed, 32 insertions(+) + +diff --git a/lib/util/abi_crosswalk.json b/lib/util/abi_crosswalk.json +index a9f2ea4..9a07647 100644 +--- a/lib/util/abi_crosswalk.json ++++ b/lib/util/abi_crosswalk.json +@@ -1503,6 +1503,10 @@ + "node_abi": 48, + "v8": "5.1" + }, ++ "6.14.4": { ++ "node_abi": 48, ++ "v8": "5.1" ++ }, + "7.0.0": { + "node_abi": 51, + "v8": "5.4" +@@ -1671,6 +1675,14 @@ + "node_abi": 57, + "v8": "6.2" + }, ++ "8.11.4": { ++ "node_abi": 57, ++ "v8": "6.2" ++ }, ++ "8.12.0": { ++ "node_abi": 57, ++ "v8": "6.2" ++ }, + "9.0.0": { + "node_abi": 59, + "v8": "6.2" +@@ -1786,5 +1798,25 @@ + "10.8.0": { + "node_abi": 64, + "v8": "6.7" ++ }, ++ "10.9.0": { ++ "node_abi": 64, ++ "v8": "6.8" ++ }, ++ "10.10.0": { ++ "node_abi": 64, ++ "v8": "6.8" ++ }, ++ "10.11.0": { ++ "node_abi": 64, ++ "v8": "6.8" ++ }, ++ "10.12.0": { ++ "node_abi": 64, ++ "v8": "6.8" ++ }, ++ "11.0.0": { ++ "node_abi": 67, ++ "v8": "7.0" + } + } +\ No newline at end of file diff --git a/native_client/javascript/package.json.in b/native_client/javascript/package.json.in index 1ef291d1..12a49dfd 100644 --- a/native_client/javascript/package.json.in +++ b/native_client/javascript/package.json.in @@ -30,13 +30,12 @@ "host" : "https://index.taskcluster.net/v1/task/project.deepspeech.tensorflow.pip.v1.0.0-warpctc.arm/artifacts/public/" }, "dependencies" : { - "node-pre-gyp": "0.6.x", + "node-pre-gyp": "0.11.x", "argparse": "1.0.x", "sox-stream": "2.0.x", "memory-stream": "0.0.3", "node-wav": "0.0.2" }, - "bundledDependencies":["node-pre-gyp"], "devDependencies": { }, "scripts": { diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index d8bff22b..c7aeebfc 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -8,6 +8,7 @@ build: ref_match: "invalid-refs-should-not-work" dependencies: [] maxRunTime: 3600 + docker_image: "ubuntu:14.04" system_setup: > true diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index b3663f04..cdf9ce02 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -16,7 +16,11 @@ nodejs: prep_7: 'echo "deb http://deb.nodesource.com/node_7.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_8: 'echo "deb http://deb.nodesource.com/node_8.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_9: 'echo "deb http://deb.nodesource.com/node_9.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' - prep_10: 'echo "deb http://deb.nodesource.com/node_10.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + packages_xenial: + apt: 'nodejs sox' + apt_pinning: '(echo "Package: nodejs" && echo "Pin: origin deb.nodesource.com" && echo "Pin-Priority: 999") > /etc/apt/preferences' + prep_10: 'echo "deb http://deb.nodesource.com/node_10.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_11: 'echo "deb http://deb.nodesource.com/node_11.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' packages_stretch: apt: 'nodejs sox' apt_pinning: '(echo "Package: nodejs" && echo "Pin: origin deb.nodesource.com" && echo "Pin-Priority: 999") > /etc/apt/preferences' @@ -27,6 +31,7 @@ nodejs: prep_8: 'echo "deb http://deb.nodesource.com/node_8.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_9: 'echo "deb http://deb.nodesource.com/node_9.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_10: 'echo "deb http://deb.nodesource.com/node_10.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_11: 'echo "deb http://deb.nodesource.com/node_11.x stretch main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' brew: # Using 'nvm' from homebrew results in installing v0.33.6 which seems to # suffer from a bug making it unable to work as of now: @@ -43,6 +48,7 @@ nodejs: prep_8: 'nvm install 8.12.0 && nvm use 8.12.0' prep_9: 'nvm install 9.11.2 && nvm use 9.11.2' prep_10: 'nvm install 10.12.0 && nvm use 10.12.0' + prep_11: 'nvm install 11.0.0 && nvm use 11.0.0' ## swig -javascript requires swig >= 3.0.7 (14.04 provides 2.0.11) ## force 3.0.10 for nodejs V8 API compatibility swig: diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index 95c141a1..54f38aed 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -8,7 +8,7 @@ build: - "notify.irc-channel.${notifications.irc}.on-failed" system_setup: > - ${nodejs.packages_trusty.prep_6} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_trusty.prep_8} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index 22ba8b2c..2f7d79e0 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -6,7 +6,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.gpu.${event.head.sha}" system_setup: > - ${nodejs.packages_trusty.prep_6} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_trusty.prep_8} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: diff --git a/taskcluster/linux-arm64-cpu-opt.yml b/taskcluster/linux-arm64-cpu-opt.yml index 49d82bcc..ba5e0ac4 100644 --- a/taskcluster/linux-arm64-cpu-opt.yml +++ b/taskcluster/linux-arm64-cpu-opt.yml @@ -12,7 +12,7 @@ build: apt-get -qq -y install gdebi git pixz && wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb && - ${nodejs.packages_trusty.prep_6} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_trusty.prep_8} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 0c78780e..76b867b3 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -12,7 +12,7 @@ build: apt-get -qq -y install gdebi git pixz && wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb && - ${nodejs.packages_trusty.prep_6} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_trusty.prep_8} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml && apt-get -qq -y install ${python.packages_trusty.apt} && ${swig.packages.install_script} system_config: diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index 37c3cf85..5c3e4041 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -27,7 +27,7 @@ then: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "ubuntu:14.04" + image: ${build.docker_image} env: $let: @@ -54,7 +54,7 @@ then: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > ${aptEc2Mirrors} && - apt-get -qq update && apt-get -qq -y install git pixz sox wget && ${extraSystemSetup} && + apt-get -qq update && apt-get -qq -y install git pixz sox sudo wget && ${extraSystemSetup} && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux} && echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && diff --git a/taskcluster/test-nodejs_10x-linux-amd64-opt.yml b/taskcluster/test-nodejs_10x-linux-amd64-opt.yml index 2a04b9f6..38d941fa 100644 --- a/taskcluster/test-nodejs_10x-linux-amd64-opt.yml +++ b/taskcluster/test-nodejs_10x-linux-amd64-opt.yml @@ -1,11 +1,12 @@ build: template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" dependencies: - "node-package" - "test-training_upstream-linux-amd64-py27mu-opt" system_setup: > - ${nodejs.packages_trusty.prep_10} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} + ${nodejs.packages_xenial.prep_10} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 10.x" metadata: diff --git a/taskcluster/test-nodejs_10x-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_10x-linux-amd64-prod_pbmodel-opt.yml index be196f73..5b66d0b5 100644 --- a/taskcluster/test-nodejs_10x-linux-amd64-prod_pbmodel-opt.yml +++ b/taskcluster/test-nodejs_10x-linux-amd64-prod_pbmodel-opt.yml @@ -1,10 +1,11 @@ build: template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" dependencies: - "node-package" system_setup: > - ${nodejs.packages_trusty.prep_10} && ${nodejs.packages_trusty.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_trusty.apt} + ${nodejs.packages_xenial.prep_10} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 10.x" metadata: diff --git a/taskcluster/test-nodejs_11x-armbian-arm64-opt.yml b/taskcluster/test-nodejs_11x-armbian-arm64-opt.yml new file mode 100644 index 00000000..2429239e --- /dev/null +++ b/taskcluster/test-nodejs_11x-armbian-arm64-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-armbian-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_stretch.prep_11} && ${nodejs.packages_stretch.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 11.x" + metadata: + name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 11.x tests" + description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v11.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_11x-darwin-amd64-opt.yml b/taskcluster/test-nodejs_11x-darwin-amd64-opt.yml new file mode 100644 index 00000000..c8bef83d --- /dev/null +++ b/taskcluster/test-nodejs_11x-darwin-amd64-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-darwin-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.brew.setup} && ${nodejs.brew.prep_11} && ${nodejs.brew.env} + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tc-node-tests.sh 11.x" + metadata: + name: "DeepSpeech OSX AMD64 CPU NodeJS 11.x tests" + description: "Testing DeepSpeech for OSX/AMD64 on NodeJS v11.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_11x-linux-amd64-opt.yml b/taskcluster/test-nodejs_11x-linux-amd64-opt.yml new file mode 100644 index 00000000..ff650d7d --- /dev/null +++ b/taskcluster/test-nodejs_11x-linux-amd64-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_11} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 11.x" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS 11.x tests" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v11.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_11x-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_11x-linux-amd64-prod_pbmodel-opt.yml new file mode 100644 index 00000000..a03d6f83 --- /dev/null +++ b/taskcluster/test-nodejs_11x-linux-amd64-prod_pbmodel-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package" + system_setup: + > + ${nodejs.packages_xenial.prep_11} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests-prod.sh 11.x" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS 11.x prod tests" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v11.x on prod model, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_11x-raspbian-rpi3-opt.yml b/taskcluster/test-nodejs_11x-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..8f128d86 --- /dev/null +++ b/taskcluster/test-nodejs_11x-raspbian-rpi3-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "node-package" + - "test-training_upstream-linux-amd64-py27mu-opt" + system_setup: + > + ${nodejs.packages_stretch.prep_11} && ${nodejs.packages_stretch.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_stretch.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/tc-node-tests.sh 11.x" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 11.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v11.x, CPU only, optimized version" diff --git a/tc-tests-utils.sh b/tc-tests-utils.sh index 59a383e3..3d5d24f5 100755 --- a/tc-tests-utils.sh +++ b/tc-tests-utils.sh @@ -36,7 +36,7 @@ model_name_mmap="$(basename -s ".pb" "${model_source}").pbmm" model_source_mmap="$(dirname "${model_source}")/${model_name_mmap}" SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-2.7.15:ucs2 2.7.15:ucs4 3.4.9:ucs4 3.5.6:ucs4 3.6.7:ucs4 3.7.1:ucs4} -SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-4.9.1 5.12.0 6.14.4 7.10.1 8.12.0 9.11.2 10.12.0} +SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-4.9.1 5.12.0 6.14.4 7.10.1 8.12.0 9.11.2 10.12.0 11.0.0} strip() { echo "$(echo $1 | sed -e 's/^[[:space:]]+//' -e 's/[[:space:]]+$//')" @@ -511,6 +511,15 @@ do_deepspeech_nodejs_build() npm update && npm install node-gyp node-pre-gyp + #FIXME: Remove when https://github.com/mapbox/node-pre-gyp/issues/421 is fixed + node_pre_gyp="$(npm root)/node-pre-gyp/" + abi_crosswalk="${node_pre_gyp}lib/util/abi_crosswalk.json" + has_node_v11=$(grep -q "11.0.0" "${abi_crosswalk}" ; echo $?) + + if [ "${has_node_v11}" -eq 1 ]; then + patch -d "${node_pre_gyp}" -p1 < native_client/javascript/node-pre-gyp_nodejs_v11.patch + fi; + export PATH="$(npm root)/.bin/:$PATH" for node in ${SUPPORTED_NODEJS_VERSIONS}; do