From 45d8f7cd617ebccafda3b0228e352dfdb545f998 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Mon, 25 May 2020 17:47:17 +0200 Subject: [PATCH 01/17] Explicitly pass filter context to multiprocessing function --- bin/import_cv2.py | 103 ++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 45 deletions(-) diff --git a/bin/import_cv2.py b/bin/import_cv2.py index 7c85c8f0..50434605 100755 --- a/bin/import_cv2.py +++ b/bin/import_cv2.py @@ -7,6 +7,7 @@ DeepSpeech.py Use "python3 import_cv2.py -h" for help """ import csv +import itertools import os import subprocess import unicodedata @@ -30,19 +31,20 @@ SAMPLE_RATE = 16000 MAX_SECS = 10 -def _preprocess_data(tsv_dir, audio_dir, space_after_every_character=False): +def _preprocess_data(tsv_dir, audio_dir, filter_obj, space_after_every_character=False): exclude = [] for dataset in ["test", "dev", "train", "validated", "other"]: - set_samples = _maybe_convert_set(dataset, tsv_dir, audio_dir, space_after_every_character) + set_samples = _maybe_convert_set(dataset, tsv_dir, audio_dir, filter_obj, space_after_every_character) if dataset in ["test", "dev"]: exclude += set_samples if dataset == "validated": - _maybe_convert_set("train-all", tsv_dir, audio_dir, space_after_every_character, - rows=set_samples, exclude=exclude) + _maybe_convert_set("train-all", tsv_dir, audio_dir, filter_obj, space_after_every_character, + filter_obj, rows=set_samples, exclude=exclude) -def one_sample(sample): - """ Take a audio file, and optionally convert it to 16kHz WAV """ +def one_sample(args): + """ Take an audio file, and optionally convert it to 16kHz WAV """ + sample, filter_obj = args mp3_filename = sample[0] if not os.path.splitext(mp3_filename.lower())[1] == ".mp3": mp3_filename += ".mp3" @@ -58,7 +60,7 @@ def one_sample(sample): ["soxi", "-s", wav_filename], stderr=subprocess.STDOUT ) ) - label = label_filter_fun(sample[1]) + label = filter_obj.filter(sample[1]) rows = [] counter = get_counter() if file_size == -1: @@ -82,7 +84,7 @@ def one_sample(sample): return (counter, rows) -def _maybe_convert_set(dataset, tsv_dir, audio_dir, space_after_every_character=None, rows=None, exclude=None): +def _maybe_convert_set(dataset, tsv_dir, audio_dir, filter_obj, space_after_every_character=None, rows=None, exclude=None): exclude_transcripts = set() exclude_speakers = set() if exclude is not None: @@ -109,7 +111,8 @@ def _maybe_convert_set(dataset, tsv_dir, audio_dir, space_after_every_character= print("Importing mp3 files...") pool = Pool() bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR) - for i, processed in enumerate(pool.imap_unordered(one_sample, samples), start=1): + samples_with_context = itertools.zip_longest(samples, [], fillvalue=filter_obj) + for i, processed in enumerate(pool.imap_unordered(one_sample, samples_with_context), start=1): counter += processed[0] rows += processed[1] bar.update(i) @@ -160,50 +163,60 @@ def _maybe_convert_wav(mp3_filename, wav_filename): except sox.core.SoxError: pass +class LabelFilter: + def __init__(self, normalize, alphabet, validate_fun): + self.normalize = normalize + self.alphabet = alphabet + self.validate_fun = validate_fun -if __name__ == "__main__": - PARSER = get_importers_parser(description="Import CommonVoice v2.0 corpora") - PARSER.add_argument("tsv_dir", help="Directory containing tsv files") - PARSER.add_argument( - "--audio_dir", - help='Directory containing the audio clips - defaults to "/clips"', - ) - PARSER.add_argument( - "--filter_alphabet", - help="Exclude samples with characters not in provided alphabet", - ) - PARSER.add_argument( - "--normalize", - action="store_true", - help="Converts diacritic characters to their base ones", - ) - PARSER.add_argument( - "--space_after_every_character", - action="store_true", - help="To help transcript join by white space", - ) - - PARAMS = PARSER.parse_args() - validate_label = get_validate_label(PARAMS) - - AUDIO_DIR = ( - PARAMS.audio_dir if PARAMS.audio_dir else os.path.join(PARAMS.tsv_dir, "clips") - ) - ALPHABET = Alphabet(PARAMS.filter_alphabet) if PARAMS.filter_alphabet else None - - def label_filter_fun(label): - if PARAMS.normalize: + def filter(self, label): + if self.normalize: label = ( unicodedata.normalize("NFKD", label.strip()) .encode("ascii", "ignore") .decode("ascii", "ignore") ) - label = validate_label(label) - if ALPHABET and label: + label = self.validate_fun(label) + if self.alphabet and label: try: - ALPHABET.encode(label) + self.alphabet.encode(label) except KeyError: label = None return label - _preprocess_data(PARAMS.tsv_dir, AUDIO_DIR, PARAMS.space_after_every_character) +def main(): + parser = get_importers_parser(description="Import CommonVoice v2.0 corpora") + parser.add_argument("tsv_dir", help="Directory containing tsv files") + parser.add_argument( + "--audio_dir", + help='Directory containing the audio clips - defaults to "/clips"', + ) + parser.add_argument( + "--filter_alphabet", + help="Exclude samples with characters not in provided alphabet", + ) + parser.add_argument( + "--normalize", + action="store_true", + help="Converts diacritic characters to their base ones", + ) + parser.add_argument( + "--space_after_every_character", + action="store_true", + help="To help transcript join by white space", + ) + + params = parser.parse_args() + validate_label = get_validate_label(params) + + audio_dir = ( + params.audio_dir if params.audio_dir else os.path.join(params.tsv_dir, "clips") + ) + alphabet = Alphabet(params.filter_alphabet) if params.filter_alphabet else None + + filter_obj = LabelFilter(params.normalize, alphabet, validate_label) + _preprocess_data(params.tsv_dir, audio_dir, filter_obj, + params.space_after_every_character) + +if __name__ == "__main__": + main() From 3d0ec01853fe9f5b73a6e35b0226e4c97c237655 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Wed, 27 May 2020 19:02:55 +0200 Subject: [PATCH 02/17] Fix typo from argument reordering --- bin/import_cv2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/import_cv2.py b/bin/import_cv2.py index 50434605..3b7e8fba 100755 --- a/bin/import_cv2.py +++ b/bin/import_cv2.py @@ -39,7 +39,7 @@ def _preprocess_data(tsv_dir, audio_dir, filter_obj, space_after_every_character exclude += set_samples if dataset == "validated": _maybe_convert_set("train-all", tsv_dir, audio_dir, filter_obj, space_after_every_character, - filter_obj, rows=set_samples, exclude=exclude) + rows=set_samples, exclude=exclude) def one_sample(args): From 0b78f4ff0130c05f783bd12d343b34192489f43c Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Fri, 29 May 2020 17:06:08 +0530 Subject: [PATCH 03/17] move unittest to TC --- taskcluster/tc-train-unittests.sh | 24 +++++++++++++++++++ ...ing-unittests_8k-linux-amd64-py36m-opt.yml | 13 ++++++++++ 2 files changed, 37 insertions(+) create mode 100644 taskcluster/tc-train-unittests.sh create mode 100644 taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml diff --git a/taskcluster/tc-train-unittests.sh b/taskcluster/tc-train-unittests.sh new file mode 100644 index 00000000..dbae276e --- /dev/null +++ b/taskcluster/tc-train-unittests.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" + +bitrate=$2 + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true +mkdir -p /tmp/train || true +mkdir -p /tmp/train_tflite || true + +virtualenv_activate "${pyalias}" "deepspeech" + +set -o pipefail +pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat +pushd ${HOME}/DeepSpeech/ds + pip install --upgrade . | cat +popd +set +o pipefail + +python -m unittest diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml new file mode 100644 index 00000000..d507fc7e --- /dev/null +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + system_setup: + > + apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m 8k" + metadata: + name: "DeepSpeech Linux AMD64 CPU 8kHz training Py3.6" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 8kHz Python 3.6, CPU only, optimized version" + \ No newline at end of file From ccca1c1fed17e5586cce43a6519d0914761c1e2c Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Fri, 29 May 2020 20:28:52 +0530 Subject: [PATCH 04/17] add tests to TC and update travis --- .travis.yml | 8 -------- taskcluster/tc-train-unittests.sh | 9 +-------- ...t-training-unittests_8k-linux-amd64-py35m-opt.yml | 12 ++++++++++++ ...t-training-unittests_8k-linux-amd64-py36m-opt.yml | 8 ++++---- ...t-training-unittests_8k-linux-amd64-py37m-opt.yml | 12 ++++++++++++ 5 files changed, 29 insertions(+), 20 deletions(-) create mode 100644 taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml create mode 100644 taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml diff --git a/.travis.yml b/.travis.yml index ca33852f..f64b1a34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,11 +20,3 @@ jobs: fi cardboardlinter --refspec $TRAVIS_BRANCH -n auto; fi - - name: python unit tests - install: - - pip install --upgrade -r requirements_tests.txt; - pip install --upgrade . - script: - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - python -m unittest; - fi diff --git a/taskcluster/tc-train-unittests.sh b/taskcluster/tc-train-unittests.sh index dbae276e..4ed6931c 100644 --- a/taskcluster/tc-train-unittests.sh +++ b/taskcluster/tc-train-unittests.sh @@ -4,13 +4,7 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" - -bitrate=$2 - mkdir -p ${TASKCLUSTER_ARTIFACTS} || true -mkdir -p /tmp/train || true -mkdir -p /tmp/train_tflite || true virtualenv_activate "${pyalias}" "deepspeech" @@ -18,7 +12,6 @@ set -o pipefail pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat pushd ${HOME}/DeepSpeech/ds pip install --upgrade . | cat + python -m unittest popd set +o pipefail - -python -m unittest diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml new file mode 100644 index 00000000..81490ec0 --- /dev/null +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml @@ -0,0 +1,12 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + system_setup: + > + apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.5.8:m" + metadata: + name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.5" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.5, CPU only, optimized version" diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml index d507fc7e..8ca12a88 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml @@ -6,8 +6,8 @@ build: > apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m 8k" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m" metadata: - name: "DeepSpeech Linux AMD64 CPU 8kHz training Py3.6" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 8kHz Python 3.6, CPU only, optimized version" - \ No newline at end of file + name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.6" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.6, CPU only, optimized version" + \ No newline at end of file diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml new file mode 100644 index 00000000..3b34eadc --- /dev/null +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml @@ -0,0 +1,12 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + system_setup: + > + apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.7.6:m" + metadata: + name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.7" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.7, CPU only, optimized version" From 73c4f3a201eae25e3ed5eeb892174d01f791b6c7 Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Fri, 29 May 2020 20:50:36 +0530 Subject: [PATCH 05/17] update tc-train-unittest.sh --- taskcluster/tc-train-unittests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskcluster/tc-train-unittests.sh b/taskcluster/tc-train-unittests.sh index 4ed6931c..4be52e90 100644 --- a/taskcluster/tc-train-unittests.sh +++ b/taskcluster/tc-train-unittests.sh @@ -4,6 +4,8 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" + mkdir -p ${TASKCLUSTER_ARTIFACTS} || true virtualenv_activate "${pyalias}" "deepspeech" From 84d2f2a5f13ce8e7c3ae0f11b8fbe8503102ca30 Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Fri, 29 May 2020 20:54:27 +0530 Subject: [PATCH 06/17] update description of test-training-unittests*yml --- .../test-training-unittests_8k-linux-amd64-py35m-opt.yml | 2 +- .../test-training-unittests_8k-linux-amd64-py36m-opt.yml | 2 +- .../test-training-unittests_8k-linux-amd64-py37m-opt.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml index 81490ec0..d33ad21d 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml @@ -9,4 +9,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.5.8:m" metadata: name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.5" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.5, CPU only, optimized version" + description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.5, for CPU only, and optimized version" diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml index 8ca12a88..195ddfe9 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml @@ -9,5 +9,5 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m" metadata: name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.6" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.6, CPU only, optimized version" + description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.6, for CPU only, and optimized version" \ No newline at end of file diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml index 3b34eadc..1f9aabc3 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml @@ -9,4 +9,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.7.6:m" metadata: name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.7" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.7, CPU only, optimized version" + description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.7, for CPU only, and optimized version" From ab2ba41c7fc79ebac02ed17abb8ce578a8d62da6 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Sat, 30 May 2020 14:45:17 +0200 Subject: [PATCH 07/17] Convert path to str to fix Python 3.5 compat --- training/deepspeech_training/util/importers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/training/deepspeech_training/util/importers.py b/training/deepspeech_training/util/importers.py index 4868e4a3..a4c3c326 100644 --- a/training/deepspeech_training/util/importers.py +++ b/training/deepspeech_training/util/importers.py @@ -42,15 +42,17 @@ def get_validate_label(args): :return: The user-supplied validate_label function :type: function """ + # Python 3.5 does not support passing a pathlib.Path to os.path.* methods if 'validate_label_locale' not in args or (args.validate_label_locale is None): print('WARNING: No --validate_label_locale specified, your might end with inconsistent dataset.') return validate_label_eng - if not os.path.exists(os.path.abspath(args.validate_label_locale)): + validate_label_locale = str(args.validate_label_locale) + if not os.path.exists(os.path.abspath(validate_label_locale)): print('ERROR: Inexistent --validate_label_locale specified. Please check.') return None - module_dir = os.path.abspath(os.path.dirname(args.validate_label_locale)) + module_dir = os.path.abspath(os.path.dirname(validate_label_locale)) sys.path.insert(1, module_dir) - fname = os.path.basename(args.validate_label_locale).replace('.py', '') + fname = os.path.basename(validate_label_locale).replace('.py', '') locale_module = importlib.import_module(fname, package=None) return locale_module.validate_label From 60397964e1b4a46814ca91bc3f9734ecb07cdee2 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 2 Jun 2020 12:29:17 +0200 Subject: [PATCH 08/17] Add some native_client build outputs to .gitignore X-DeepSpeech: NOBUILD --- .gitignore | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 560e67c1..a44fa2fb 100644 --- a/.gitignore +++ b/.gitignore @@ -14,14 +14,21 @@ /native_client/build /native_client/*.egg-info /native_client/dist +/native_client/deepspeech +/native_client/ds-swig +/native_client/libdeepspeech.so /native_client/node_modules -/native_client/python/model.py -/native_client/python/utils.py -/native_client/python/model_wrap.cpp -/native_client/python/utils_wrap.cpp /native_client/javascript/build +/native_client/javascript/lib +/native_client/javascript/package.json +/native_client/javascript/package-lock.json /native_client/javascript/client.js /native_client/javascript/deepspeech_wrap.cxx +/native_client/javascript/node_modules +/native_client/python/MANIFEST.in +/native_client/python/dist +/native_client/python/impl.py +/native_client/python/impl_wrap.cpp /doc/.build/ /doc/xml-c/ /doc/xml-java/ From 00577873ce8debf412f4a30f2b075dc9e44e7bc8 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 2 Jun 2020 13:34:28 +0200 Subject: [PATCH 09/17] Update Homebrew --- taskcluster/homebrew-build.sh | 2 +- taskcluster/tc-all-vars.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/homebrew-build.sh b/taskcluster/homebrew-build.sh index 69dd1628..c04eba0c 100755 --- a/taskcluster/homebrew-build.sh +++ b/taskcluster/homebrew-build.sh @@ -29,7 +29,7 @@ do_prepare_homebrew() export PATH=${_brew_instance}/bin:$PATH export HOMEBREW_LOGS="${_brew_instance}/homebrew.logs/" export HOMEBREW_CACHE="${_brew_instance}/homebrew.cache/" - export BREW_FORMULAS_COMMIT=c3f06e4f17853bea8c35742923a9c43d3a244d35 + export BREW_FORMULAS_COMMIT=ddd39cf1b71452bfe9c5f17f45cc0118796b20d3 # Never fail on pre-existing homebrew/ directory mkdir -p "${_brew_instance}" || true diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 6cdc6e7c..67f86bf4 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -25,7 +25,7 @@ if [ "${OS}" = "Darwin" ]; then export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root" export HOMEBREW_NO_AUTO_UPDATE=1 - export BREW_URL=https://github.com/Homebrew/brew/tarball/2.1.14 + export BREW_URL=https://github.com/Homebrew/brew/tarball/2.2.17 export BUILDS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-builds" export TESTS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-tests" From f925dd9fc8da44cec3a38775185e2be75641b873 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 2 Jun 2020 13:47:51 +0200 Subject: [PATCH 10/17] Fix Homebrew checks --- taskcluster/homebrew-build.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/taskcluster/homebrew-build.sh b/taskcluster/homebrew-build.sh index c04eba0c..d0f9df0e 100755 --- a/taskcluster/homebrew-build.sh +++ b/taskcluster/homebrew-build.sh @@ -40,7 +40,7 @@ do_prepare_homebrew() curl -L ${BREW_URL} | tar xz --strip 1 -C "${_brew_instance}" fi; - check_homebrew + check_homebrew "${_brew_instance}" # Force an upgrade to fetch formulae brew search openssl @@ -53,14 +53,16 @@ do_prepare_homebrew() check_homebrew() { - echo "local brew prefix ..." - local_prefix=$(brew --prefix) - echo "${local_prefix}" + local _expected_prefix=$1 - if [ "${BUILDS_BREW}" != "${local_prefix}" ]; then + echo "local brew prefix ..." + local _local_prefix=$(brew --prefix) + echo "${_local_prefix}" + + if [ "${_expected_prefix}" != "${_local_prefix}" ]; then echo "Weird state:" - echo "BUILDS_BREW=${BUILDS_BREW}" - echo "local_prefix=${local_prefix}" + echo "_expected_prefix=${_expected_prefix}" + echo "_local_prefix=${_local_prefix}" exit 1 fi; } From dc8dbbd398fe978adec88a32d33efffc43b27c76 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 2 Jun 2020 12:10:19 +0200 Subject: [PATCH 11/17] Add NodeJS v14 --- native_client/javascript/package.json.in | 2 +- taskcluster/.shared.yml | 16 ++++++++++------ taskcluster/homebrew_builds-darwin-amd64.yml | 4 ++-- taskcluster/homebrew_tests-darwin-amd64.yml | 4 ++-- taskcluster/tc-all-vars.sh | 2 +- .../test-nodejs_14x-armbian-arm64-opt.yml | 14 ++++++++++++++ taskcluster/test-nodejs_14x-darwin-amd64-opt.yml | 15 +++++++++++++++ .../test-nodejs_14x-raspbian-rpi3-opt.yml | 14 ++++++++++++++ taskcluster/test-nodejs_14x-win-amd64-opt.yml | 14 ++++++++++++++ .../test-nodejs_14x_16k-linux-amd64-opt.yml | 15 +++++++++++++++ ...dejs_14x_16k-linux-amd64-prod_pbmodel-opt.yml | 13 +++++++++++++ ...dejs_14x_16k_multiarchpkg-linux-amd64-opt.yml | 15 +++++++++++++++ ...multiarchpkg-linux-amd64-prod_pbmodel-opt.yml | 13 +++++++++++++ ...ejs_14x_16k_multiarchpkg-linux-tflite-opt.yml | 15 +++++++++++++++ ...odejs_14x_16k_tflite-linux-amd64-prod-opt.yml | 13 +++++++++++++ .../test-nodejs_14x_8k-linux-amd64-opt.yml | 15 +++++++++++++++ ...odejs_14x_8k-linux-amd64-prod_pbmodel-opt.yml | 13 +++++++++++++ ...odejs_14x_8k_multiarchpkg-linux-amd64-opt.yml | 15 +++++++++++++++ ...multiarchpkg-linux-amd64-prod_pbmodel-opt.yml | 13 +++++++++++++ ...dejs_14x_8k_multiarchpkg-linux-tflite-opt.yml | 15 +++++++++++++++ ...nodejs_14x_8k_tflite-linux-amd64-prod-opt.yml | 13 +++++++++++++ ...nodejs_14x_multiarchpkg-armbian-arm64-opt.yml | 14 ++++++++++++++ ...-nodejs_14x_multiarchpkg-darwin-amd64-opt.yml | 15 +++++++++++++++ ...nodejs_14x_multiarchpkg-darwin-tflite-opt.yml | 15 +++++++++++++++ ...nodejs_14x_multiarchpkg-raspbian-rpi3-opt.yml | 14 ++++++++++++++ ...est-nodejs_14x_multiarchpkg-win-amd64-opt.yml | 14 ++++++++++++++ ...test-nodejs_14x_multiarchpkg-win-cuda-opt.yml | 14 ++++++++++++++ ...st-nodejs_14x_multiarchpkg-win-tflite-opt.yml | 14 ++++++++++++++ 28 files changed, 341 insertions(+), 12 deletions(-) create mode 100644 taskcluster/test-nodejs_14x-armbian-arm64-opt.yml create mode 100644 taskcluster/test-nodejs_14x-darwin-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_14x-raspbian-rpi3-opt.yml create mode 100644 taskcluster/test-nodejs_14x-win-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_14x_16k-linux-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_14x_16k-linux-amd64-prod_pbmodel-opt.yml create mode 100644 taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-amd64-prod_pbmodel-opt.yml create mode 100644 taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-tflite-opt.yml create mode 100644 taskcluster/test-nodejs_14x_16k_tflite-linux-amd64-prod-opt.yml create mode 100644 taskcluster/test-nodejs_14x_8k-linux-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_14x_8k-linux-amd64-prod_pbmodel-opt.yml create mode 100644 taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-amd64-prod_pbmodel-opt.yml create mode 100644 taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-tflite-opt.yml create mode 100644 taskcluster/test-nodejs_14x_8k_tflite-linux-amd64-prod-opt.yml create mode 100644 taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml create mode 100644 taskcluster/test-nodejs_14x_multiarchpkg-darwin-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_14x_multiarchpkg-darwin-tflite-opt.yml create mode 100644 taskcluster/test-nodejs_14x_multiarchpkg-raspbian-rpi3-opt.yml create mode 100644 taskcluster/test-nodejs_14x_multiarchpkg-win-amd64-opt.yml create mode 100644 taskcluster/test-nodejs_14x_multiarchpkg-win-cuda-opt.yml create mode 100644 taskcluster/test-nodejs_14x_multiarchpkg-win-tflite-opt.yml diff --git a/native_client/javascript/package.json.in b/native_client/javascript/package.json.in index 00079851..09dad3a5 100644 --- a/native_client/javascript/package.json.in +++ b/native_client/javascript/package.json.in @@ -32,7 +32,7 @@ "host" : "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.v1.0.0-warpctc.arm/artifacts/public/" }, "dependencies" : { - "node-pre-gyp": "0.14.x", + "node-pre-gyp": "0.15.x", "argparse": "1.0.x", "sox-stream": "2.0.x", "memory-stream": "0.0.3", diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index a5492b82..c0ff888c 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -29,6 +29,7 @@ nodejs: prep_11: 'echo "deb http://deb.nodesource.com/node_11.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_12: 'echo "deb http://deb.nodesource.com/node_12.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_13: 'echo "deb http://deb.nodesource.com/node_13.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_14: 'echo "deb http://deb.nodesource.com/node_14.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' packages_docs_bionic: apt: 'nodejs' apt_pinning: '(echo "Package: nodejs" && echo "Pin: origin deb.nodesource.com" && echo "Pin-Priority: 999") > /etc/apt/preferences' @@ -41,26 +42,29 @@ nodejs: prep_11: 'echo "deb http://deb.nodesource.com/node_11.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_12: 'echo "deb http://deb.nodesource.com/node_12.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_13: 'echo "deb http://deb.nodesource.com/node_13.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_14: 'echo "deb http://deb.nodesource.com/node_14.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' brew: prep_10: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 10.18.1' prep_11: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 11.15.0' prep_12: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 12.8.1' prep_13: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 13.1.0' + prep_14: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 14.3.0' win: prep_10: '/usr/bin/wget.exe https://nodejs.org/dist/v10.18.1/node-v10.18.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v10.18.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v10.18.1-win-x64/:$PATH' prep_11: '/usr/bin/wget.exe https://nodejs.org/dist/v11.15.0/node-v11.15.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v11.15.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v11.15.0-win-x64/:$PATH' prep_12: '/usr/bin/wget.exe https://nodejs.org/dist/v12.14.1/node-v12.14.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v12.14.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v12.14.1-win-x64/:$PATH' prep_13: '/usr/bin/wget.exe https://nodejs.org/dist/v13.6.0/node-v13.6.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v13.6.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v13.6.0-win-x64/:$PATH' + prep_14: '/usr/bin/wget.exe https://nodejs.org/dist/v14.3.0/node-v14.3.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v14.3.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v14.3.0-win-x64/:$PATH' system: node_gyp_cache: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.2/artifacts/public/node-gyp-cache.tar.gz' - namespace: 'project.deepspeech.node-gyp-cache.2' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.3/artifacts/public/node-gyp-cache.tar.gz' + namespace: 'project.deepspeech.node-gyp-cache.3' homebrew_builds: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.4/artifacts/public/homebrew_builds.tar.gz' - namespace: 'project.deepspeech.homebrew_builds.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz' + namespace: 'project.deepspeech.homebrew_builds.5' homebrew_tests: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz' - namespace: 'project.deepspeech.homebrew_tests.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.5/artifacts/public/homebrew_tests.tar.gz' + namespace: 'project.deepspeech.homebrew_tests.5' android_cache: arm64_v8a: android_24: diff --git a/taskcluster/homebrew_builds-darwin-amd64.yml b/taskcluster/homebrew_builds-darwin-amd64.yml index 258f9ded..4dac3a13 100644 --- a/taskcluster/homebrew_builds-darwin-amd64.yml +++ b/taskcluster/homebrew_builds-darwin-amd64.yml @@ -1,8 +1,8 @@ build: template_file: homebrew-darwin-opt-base.tyml homebrew: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.4/artifacts/public/homebrew_builds.tar.gz' - namespace: 'project.deepspeech.homebrew_builds.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz' + namespace: 'project.deepspeech.homebrew_builds.5' scripts: build: "taskcluster/homebrew-build.sh --builds" package: "taskcluster/homebrew-package.sh --builds" diff --git a/taskcluster/homebrew_tests-darwin-amd64.yml b/taskcluster/homebrew_tests-darwin-amd64.yml index 68e61bdd..1b9fc38c 100644 --- a/taskcluster/homebrew_tests-darwin-amd64.yml +++ b/taskcluster/homebrew_tests-darwin-amd64.yml @@ -1,8 +1,8 @@ build: template_file: homebrew-darwin-opt-base.tyml homebrew: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz' - namespace: 'project.deepspeech.homebrew_tests.4' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.5/artifacts/public/homebrew_tests.tar.gz' + namespace: 'project.deepspeech.homebrew_tests.5' scripts: build: "taskcluster/homebrew-build.sh --tests" package: "taskcluster/homebrew-package.sh --tests" diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 67f86bf4..37a74ff3 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -87,5 +87,5 @@ SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-3.5.8:ucs2 3.6.10:ucs2 3. # When updating NodeJS / ElectronJS supported versions, do not forget to increment # deepspeech.node-gyp-cache. in both `system.node_gyp_cache` (taskcluster/.shared.yml) # and route index (taskcluster/node-gyp-cache.yml) to ensure the cache is updated -SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-10.18.1 11.15.0 12.8.1 13.1.0} +SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-10.18.1 11.15.0 12.8.1 13.1.0 14.0.0} SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1} diff --git a/taskcluster/test-nodejs_14x-armbian-arm64-opt.yml b/taskcluster/test-nodejs_14x-armbian-arm64-opt.yml new file mode 100644 index 00000000..d576c7ca --- /dev/null +++ b/taskcluster/test-nodejs_14x-armbian-arm64-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-armbian-opt-base.tyml + dependencies: + - "linux-arm64-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_buster.prep_14} && ${nodejs.packages_buster.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_buster.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 14.x tests" + description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x-darwin-amd64-opt.yml b/taskcluster/test-nodejs_14x-darwin-amd64-opt.yml new file mode 100644 index 00000000..8cf3f96f --- /dev/null +++ b/taskcluster/test-nodejs_14x-darwin-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-darwin-opt-base.tyml + dependencies: + - "darwin-amd64-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.brew.prep_14} + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech OSX AMD64 CPU NodeJS 14.x tests" + description: "Testing DeepSpeech for OSX/AMD64 on NodeJS v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x-raspbian-rpi3-opt.yml b/taskcluster/test-nodejs_14x-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..22ab6195 --- /dev/null +++ b/taskcluster/test-nodejs_14x-raspbian-rpi3-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "linux-rpi3-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_buster.prep_14} && ${nodejs.packages_buster.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_buster.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 14.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x-win-amd64-opt.yml b/taskcluster/test-nodejs_14x-win-amd64-opt.yml new file mode 100644 index 00000000..57a4397f --- /dev/null +++ b/taskcluster/test-nodejs_14x-win-amd64-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_14} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech Windows AMD64 CPU NodeJS 14.x tests" + description: "Testing DeepSpeech for Windows/AMD64 on NodeJS v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x_16k-linux-amd64-opt.yml b/taskcluster/test-nodejs_14x_16k-linux-amd64-opt.yml new file mode 100644 index 00000000..84b77785 --- /dev/null +++ b/taskcluster/test-nodejs_14x_16k-linux-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "linux-amd64-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS 14.x tests (16kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x, CPU only, optimized version (16kHz)" diff --git a/taskcluster/test-nodejs_14x_16k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_14x_16k-linux-amd64-prod_pbmodel-opt.yml new file mode 100644 index 00000000..0d195a45 --- /dev/null +++ b/taskcluster/test-nodejs_14x_16k-linux-amd64-prod_pbmodel-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "linux-amd64-cpu-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests-prod.sh 14.x 16k" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS 14.x prod tests (16kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x on prod model, CPU only, optimized version (16kHz)" diff --git a/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-amd64-opt.yml b/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-amd64-opt.yml new file mode 100644 index 00000000..4c1c5fa7 --- /dev/null +++ b/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package-cpu" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS MultiArch Package 14.x tests (16kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x, CPU only, optimized version (16kHz)" diff --git a/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-amd64-prod_pbmodel-opt.yml new file mode 100644 index 00000000..2f51e579 --- /dev/null +++ b/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-amd64-prod_pbmodel-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package-cpu" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests-prod.sh 14.x 16k" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS MultiArch Package 14.x prod tests (16kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x on prod model, CPU only, optimized version (16kHz)" diff --git a/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-tflite-opt.yml b/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-tflite-opt.yml new file mode 100644 index 00000000..ef8b1f66 --- /dev/null +++ b/taskcluster/test-nodejs_14x_16k_multiarchpkg-linux-tflite-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package-tflite" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech Linux AMD64 TFLite NodeJS MultiArch Package 14.x tests (16kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x, TFLite only, optimized version (16kHz)" diff --git a/taskcluster/test-nodejs_14x_16k_tflite-linux-amd64-prod-opt.yml b/taskcluster/test-nodejs_14x_16k_tflite-linux-amd64-prod-opt.yml new file mode 100644 index 00000000..57e2d23a --- /dev/null +++ b/taskcluster/test-nodejs_14x_16k_tflite-linux-amd64-prod-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "linux-amd64-tflite-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 14.x 16k" + metadata: + name: "DeepSpeech Linux AMD64 TFLite NodeJS 14.x prod tests (16kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x on prod model, TFLite, optimized version (16kHz)" diff --git a/taskcluster/test-nodejs_14x_8k-linux-amd64-opt.yml b/taskcluster/test-nodejs_14x_8k-linux-amd64-opt.yml new file mode 100644 index 00000000..9f064566 --- /dev/null +++ b/taskcluster/test-nodejs_14x_8k-linux-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "linux-amd64-cpu-opt" + - "test-training_8k-linux-amd64-py36m-opt" + test_model_task: "test-training_8k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 8k" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS 14.x tests (8kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x, CPU only, optimized version (8kHz)" diff --git a/taskcluster/test-nodejs_14x_8k-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_14x_8k-linux-amd64-prod_pbmodel-opt.yml new file mode 100644 index 00000000..a90c7dfe --- /dev/null +++ b/taskcluster/test-nodejs_14x_8k-linux-amd64-prod_pbmodel-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "linux-amd64-cpu-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests-prod.sh 14.x 8k" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS 14.x prod tests (8kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x on prod model, CPU only, optimized version (8kHz)" diff --git a/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-amd64-opt.yml b/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-amd64-opt.yml new file mode 100644 index 00000000..595bf5dd --- /dev/null +++ b/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package-cpu" + - "test-training_8k-linux-amd64-py36m-opt" + test_model_task: "test-training_8k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 8k" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS MultiArch Package 14.x tests (8kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x, CPU only, optimized version (8kHz)" diff --git a/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-amd64-prod_pbmodel-opt.yml b/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-amd64-prod_pbmodel-opt.yml new file mode 100644 index 00000000..a66a9345 --- /dev/null +++ b/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-amd64-prod_pbmodel-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package-cpu" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests-prod.sh 14.x 8k" + metadata: + name: "DeepSpeech Linux AMD64 CPU NodeJS MultiArch Package 14.x prod tests (8kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x on prod model, CPU only, optimized version (8kHz)" diff --git a/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-tflite-opt.yml b/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-tflite-opt.yml new file mode 100644 index 00000000..3f7db2d5 --- /dev/null +++ b/taskcluster/test-nodejs_14x_8k_multiarchpkg-linux-tflite-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "node-package-tflite" + - "test-training_8k-linux-amd64-py36m-opt" + test_model_task: "test-training_8k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 8k" + metadata: + name: "DeepSpeech Linux AMD64 TFLite NodeJS MultiArch Package 14.x tests (8kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x, TFLite only, optimized version (8kHz)" diff --git a/taskcluster/test-nodejs_14x_8k_tflite-linux-amd64-prod-opt.yml b/taskcluster/test-nodejs_14x_8k_tflite-linux-amd64-prod-opt.yml new file mode 100644 index 00000000..01da67be --- /dev/null +++ b/taskcluster/test-nodejs_14x_8k_tflite-linux-amd64-prod-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "linux-amd64-tflite-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 14.x 8k" + metadata: + name: "DeepSpeech Linux AMD64 TFLite NodeJS 14.x prod tests (8kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x on prod model, TFLite, optimized version (8kHz)" diff --git a/taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml b/taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml new file mode 100644 index 00000000..2f7b792e --- /dev/null +++ b/taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-armbian-opt-base.tyml + dependencies: + - "node-package-cpu" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_buster.prep_14} && ${nodejs.packages_buster.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_buster.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS MultiArch Package 14.x tests" + description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS MultiArch Package v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x_multiarchpkg-darwin-amd64-opt.yml b/taskcluster/test-nodejs_14x_multiarchpkg-darwin-amd64-opt.yml new file mode 100644 index 00000000..3c7a9cbf --- /dev/null +++ b/taskcluster/test-nodejs_14x_multiarchpkg-darwin-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-darwin-opt-base.tyml + dependencies: + - "node-package-cpu" + - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.brew.prep_14} + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech OSX AMD64 CPU NodeJS MultiArch Package 14.x tests" + description: "Testing DeepSpeech for OSX/AMD64 on NodeJS MultiArch Package v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x_multiarchpkg-darwin-tflite-opt.yml b/taskcluster/test-nodejs_14x_multiarchpkg-darwin-tflite-opt.yml new file mode 100644 index 00000000..9d5aa2f1 --- /dev/null +++ b/taskcluster/test-nodejs_14x_multiarchpkg-darwin-tflite-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-darwin-opt-base.tyml + dependencies: + - "node-package-tflite" + - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.brew.prep_14} + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech OSX AMD64 TFLite NodeJS MultiArch Package 14.x tests" + description: "Testing DeepSpeech for OSX/AMD64 on NodeJS MultiArch Package v14.x, TFLite only, optimized version" diff --git a/taskcluster/test-nodejs_14x_multiarchpkg-raspbian-rpi3-opt.yml b/taskcluster/test-nodejs_14x_multiarchpkg-raspbian-rpi3-opt.yml new file mode 100644 index 00000000..6fb29de1 --- /dev/null +++ b/taskcluster/test-nodejs_14x_multiarchpkg-raspbian-rpi3-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-raspbian-opt-base.tyml + dependencies: + - "node-package-cpu" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_buster.prep_14} && ${nodejs.packages_buster.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_buster.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS MultiArch Package 14.x tests" + description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS MultiArch Package v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x_multiarchpkg-win-amd64-opt.yml b/taskcluster/test-nodejs_14x_multiarchpkg-win-amd64-opt.yml new file mode 100644 index 00000000..bd7de4d9 --- /dev/null +++ b/taskcluster/test-nodejs_14x_multiarchpkg-win-amd64-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "node-package-cpu" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_14} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech Windows AMD64 CPU NodeJS MultiArch Package 14.x tests" + description: "Testing DeepSpeech for Windows/AMD64 on NodeJS MultiArch Package v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x_multiarchpkg-win-cuda-opt.yml b/taskcluster/test-nodejs_14x_multiarchpkg-win-cuda-opt.yml new file mode 100644 index 00000000..733afc8f --- /dev/null +++ b/taskcluster/test-nodejs_14x_multiarchpkg-win-cuda-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-win-cuda-opt-base.tyml + dependencies: + - "node-package-gpu" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_14} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k cuda" + metadata: + name: "DeepSpeech Windows AMD64 CUDA NodeJS MultiArch Package 14.x tests" + description: "Testing DeepSpeech for Windows/AMD64 on NodeJS MultiArch Package v14.x, CUDA, optimized version" diff --git a/taskcluster/test-nodejs_14x_multiarchpkg-win-tflite-opt.yml b/taskcluster/test-nodejs_14x_multiarchpkg-win-tflite-opt.yml new file mode 100644 index 00000000..f2721d6c --- /dev/null +++ b/taskcluster/test-nodejs_14x_multiarchpkg-win-tflite-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "node-package-tflite" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_14} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" + metadata: + name: "DeepSpeech Windows AMD64 TFLite NodeJS MultiArch Package 14.x tests" + description: "Testing DeepSpeech for Windows/AMD64 on NodeJS MultiArch Package v14.x, TFLite only, optimized version" From cdeb933c0b8b0a4f63bf5003db1af31ebf7fc5e5 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 2 Jun 2020 12:20:39 +0200 Subject: [PATCH 12/17] Add ElectronJS v9.0 --- taskcluster/tc-all-vars.sh | 5 +++-- .../test-electronjs_v9.0-darwin-amd64-opt.yml | 15 +++++++++++++++ .../test-electronjs_v9.0-win-amd64-opt.yml | 14 ++++++++++++++ .../test-electronjs_v9.0_16k-linux-amd64-opt.yml | 15 +++++++++++++++ .../test-electronjs_v9.0_8k-linux-amd64-opt.yml | 15 +++++++++++++++ ...electronjs_v9.0_multiarchpkg-win-amd64-opt.yml | 14 ++++++++++++++ ...-electronjs_v9.0_multiarchpkg-win-cuda-opt.yml | 14 ++++++++++++++ ..._v9.0_multiarchpkg-win-tflite-opt.yml.disabled | 14 ++++++++++++++ 8 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 taskcluster/test-electronjs_v9.0-darwin-amd64-opt.yml create mode 100644 taskcluster/test-electronjs_v9.0-win-amd64-opt.yml create mode 100644 taskcluster/test-electronjs_v9.0_16k-linux-amd64-opt.yml create mode 100644 taskcluster/test-electronjs_v9.0_8k-linux-amd64-opt.yml create mode 100644 taskcluster/test-electronjs_v9.0_multiarchpkg-win-amd64-opt.yml create mode 100644 taskcluster/test-electronjs_v9.0_multiarchpkg-win-cuda-opt.yml create mode 100644 taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml.disabled diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 37a74ff3..a73dd217 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -87,5 +87,6 @@ SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-3.5.8:ucs2 3.6.10:ucs2 3. # When updating NodeJS / ElectronJS supported versions, do not forget to increment # deepspeech.node-gyp-cache. in both `system.node_gyp_cache` (taskcluster/.shared.yml) # and route index (taskcluster/node-gyp-cache.yml) to ensure the cache is updated -SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-10.18.1 11.15.0 12.8.1 13.1.0 14.0.0} -SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1} +# Also, always target first major branch release to ensure proper binary compatibility +SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-10.0.0 11.0.0 12.7.0 13.0.0 14.0.0} +SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1} diff --git a/taskcluster/test-electronjs_v9.0-darwin-amd64-opt.yml b/taskcluster/test-electronjs_v9.0-darwin-amd64-opt.yml new file mode 100644 index 00000000..2c9ce143 --- /dev/null +++ b/taskcluster/test-electronjs_v9.0-darwin-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-darwin-opt-base.tyml + dependencies: + - "darwin-amd64-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.brew.prep_12} + args: + tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k" + metadata: + name: "DeepSpeech OSX AMD64 CPU ElectronJS v9.0 tests" + description: "Testing DeepSpeech for OSX/AMD64 on ElectronJS v9.0, CPU only, optimized version" diff --git a/taskcluster/test-electronjs_v9.0-win-amd64-opt.yml b/taskcluster/test-electronjs_v9.0-win-amd64-opt.yml new file mode 100644 index 00000000..66bd7b1f --- /dev/null +++ b/taskcluster/test-electronjs_v9.0-win-amd64-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "win-amd64-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_12} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k" + metadata: + name: "DeepSpeech Windows AMD64 CPU ElectronJS v9.0 tests" + description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS v9.0, CPU only, optimized version" diff --git a/taskcluster/test-electronjs_v9.0_16k-linux-amd64-opt.yml b/taskcluster/test-electronjs_v9.0_16k-linux-amd64-opt.yml new file mode 100644 index 00000000..15f53576 --- /dev/null +++ b/taskcluster/test-electronjs_v9.0_16k-linux-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "linux-amd64-cpu-opt" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} ${electronjs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k" + metadata: + name: "DeepSpeech Linux AMD64 CPU ElectronJS v9.0 tests (16kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on ElectronJS v9.0, CPU only, optimized version (16kHz)" diff --git a/taskcluster/test-electronjs_v9.0_8k-linux-amd64-opt.yml b/taskcluster/test-electronjs_v9.0_8k-linux-amd64-opt.yml new file mode 100644 index 00000000..4704d64d --- /dev/null +++ b/taskcluster/test-electronjs_v9.0_8k-linux-amd64-opt.yml @@ -0,0 +1,15 @@ +build: + template_file: test-linux-opt-base.tyml + docker_image: "ubuntu:16.04" + dependencies: + - "linux-amd64-cpu-opt" + - "test-training_8k-linux-amd64-py36m-opt" + test_model_task: "test-training_8k-linux-amd64-py36m-opt" + system_setup: + > + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} ${electronjs.packages_xenial.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 8k" + metadata: + name: "DeepSpeech Linux AMD64 CPU ElectronJS v9.0 tests (8kHz)" + description: "Testing DeepSpeech for Linux/AMD64 on ElectronJS v9.0, CPU only, optimized version (8kHz)" diff --git a/taskcluster/test-electronjs_v9.0_multiarchpkg-win-amd64-opt.yml b/taskcluster/test-electronjs_v9.0_multiarchpkg-win-amd64-opt.yml new file mode 100644 index 00000000..0b6d51f3 --- /dev/null +++ b/taskcluster/test-electronjs_v9.0_multiarchpkg-win-amd64-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "node-package-cpu" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_12} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k" + metadata: + name: "DeepSpeech Windows AMD64 CPU ElectronJS MultiArch Package v9.0 tests" + description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v9.0, CPU only, optimized version" diff --git a/taskcluster/test-electronjs_v9.0_multiarchpkg-win-cuda-opt.yml b/taskcluster/test-electronjs_v9.0_multiarchpkg-win-cuda-opt.yml new file mode 100644 index 00000000..957a0281 --- /dev/null +++ b/taskcluster/test-electronjs_v9.0_multiarchpkg-win-cuda-opt.yml @@ -0,0 +1,14 @@ +build: + template_file: test-win-cuda-opt-base.tyml + dependencies: + - "node-package-gpu" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_12} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k cuda" + metadata: + name: "DeepSpeech Windows AMD64 CUDA ElectronJS MultiArch Package v9.0 tests" + description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v9.0, CUDA, optimized version" diff --git a/taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml.disabled b/taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml.disabled new file mode 100644 index 00000000..0813a7a0 --- /dev/null +++ b/taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml.disabled @@ -0,0 +1,14 @@ +build: + template_file: test-win-opt-base.tyml + dependencies: + - "node-package-tflite" + - "test-training_16k-linux-amd64-py36m-opt" + test_model_task: "test-training_16k-linux-amd64-py36m-opt" + system_setup: + > + ${system.sox_win} && ${nodejs.win.prep_12} + args: + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k" + metadata: + name: "DeepSpeech Windows AMD64 TFLite ElectronJS MultiArch Package v9.0 tests" + description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v9.0, TFLite only, optimized version" From aa4c746899b4bab3ed62813b44120c385adda1ea Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 2 Jun 2020 12:15:30 +0200 Subject: [PATCH 13/17] Maximize binary compatibility --- taskcluster/.shared.yml | 14 +++++++------- taskcluster/homebrew-build.sh | 2 +- taskcluster/homebrew_tests-darwin-amd64.yml | 4 ++-- taskcluster/node-gyp-populate.sh | 2 +- taskcluster/tc-all-vars.sh | 8 ++++++-- taskcluster/tc-build-utils.sh | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index c0ff888c..24c73887 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -44,16 +44,16 @@ nodejs: prep_13: 'echo "deb http://deb.nodesource.com/node_13.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_14: 'echo "deb http://deb.nodesource.com/node_14.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' brew: - prep_10: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 10.18.1' + prep_10: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 10.20.1' prep_11: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 11.15.0' - prep_12: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 12.8.1' - prep_13: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 13.1.0' + prep_12: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 12.17.0' + prep_13: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 13.14.0' prep_14: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 14.3.0' win: - prep_10: '/usr/bin/wget.exe https://nodejs.org/dist/v10.18.1/node-v10.18.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v10.18.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v10.18.1-win-x64/:$PATH' + prep_10: '/usr/bin/wget.exe https://nodejs.org/dist/v10.20.1/node-v10.20.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v10.20.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v10.20.1-win-x64/:$PATH' prep_11: '/usr/bin/wget.exe https://nodejs.org/dist/v11.15.0/node-v11.15.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v11.15.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v11.15.0-win-x64/:$PATH' prep_12: '/usr/bin/wget.exe https://nodejs.org/dist/v12.14.1/node-v12.14.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v12.14.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v12.14.1-win-x64/:$PATH' - prep_13: '/usr/bin/wget.exe https://nodejs.org/dist/v13.6.0/node-v13.6.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v13.6.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v13.6.0-win-x64/:$PATH' + prep_13: '/usr/bin/wget.exe https://nodejs.org/dist/v13.14.0/node-v13.14.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v13.14.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v13.14.0-win-x64/:$PATH' prep_14: '/usr/bin/wget.exe https://nodejs.org/dist/v14.3.0/node-v14.3.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v14.3.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v14.3.0-win-x64/:$PATH' system: node_gyp_cache: @@ -63,8 +63,8 @@ system: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz' namespace: 'project.deepspeech.homebrew_builds.5' homebrew_tests: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.5/artifacts/public/homebrew_tests.tar.gz' - namespace: 'project.deepspeech.homebrew_tests.5' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.6/artifacts/public/homebrew_tests.tar.gz' + namespace: 'project.deepspeech.homebrew_tests.6' android_cache: arm64_v8a: android_24: diff --git a/taskcluster/homebrew-build.sh b/taskcluster/homebrew-build.sh index d0f9df0e..6cdda80a 100755 --- a/taskcluster/homebrew-build.sh +++ b/taskcluster/homebrew-build.sh @@ -92,7 +92,7 @@ prepare_homebrew_tests() install_pkg_homebrew "nvm" source "${TESTS_BREW}/opt/nvm/nvm.sh" - for node_ver in ${SUPPORTED_NODEJS_VERSIONS}; + for node_ver in ${SUPPORTED_NODEJS_TESTS_VERSIONS}; do nvm install ${node_ver} done; diff --git a/taskcluster/homebrew_tests-darwin-amd64.yml b/taskcluster/homebrew_tests-darwin-amd64.yml index 1b9fc38c..fc9637f2 100644 --- a/taskcluster/homebrew_tests-darwin-amd64.yml +++ b/taskcluster/homebrew_tests-darwin-amd64.yml @@ -1,8 +1,8 @@ build: template_file: homebrew-darwin-opt-base.tyml homebrew: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.5/artifacts/public/homebrew_tests.tar.gz' - namespace: 'project.deepspeech.homebrew_tests.5' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.6/artifacts/public/homebrew_tests.tar.gz' + namespace: 'project.deepspeech.homebrew_tests.6' scripts: build: "taskcluster/homebrew-build.sh --tests" package: "taskcluster/homebrew-package.sh --tests" diff --git a/taskcluster/node-gyp-populate.sh b/taskcluster/node-gyp-populate.sh index ebc0e47a..7043d332 100755 --- a/taskcluster/node-gyp-populate.sh +++ b/taskcluster/node-gyp-populate.sh @@ -16,7 +16,7 @@ mkdir -p ${devDir} node-gyp list ${devDir} -for node in ${SUPPORTED_NODEJS_VERSIONS}; do +for node in ${SUPPORTED_NODEJS_BUILD_VERSIONS}; do node-gyp install --devdir ${devDir} \ --target=$node mkdir ${devDir}/${node}/x64/ || true diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index a73dd217..74887bdc 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -87,6 +87,10 @@ SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-3.5.8:ucs2 3.6.10:ucs2 3. # When updating NodeJS / ElectronJS supported versions, do not forget to increment # deepspeech.node-gyp-cache. in both `system.node_gyp_cache` (taskcluster/.shared.yml) # and route index (taskcluster/node-gyp-cache.yml) to ensure the cache is updated -# Also, always target first major branch release to ensure proper binary compatibility -SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-10.0.0 11.0.0 12.7.0 13.0.0 14.0.0} +# +# Also, builds should always target first major branch release to ensure proper binary compatibility +# and tests should always target latest major branch release version +SUPPORTED_NODEJS_BUILD_VERSIONS=${SUPPORTED_NODEJS_BUILD_VERSIONS:-10.0.0 11.0.0 12.7.0 13.0.0 14.0.0} +SUPPORTED_NODEJS_TESTS_VERSIONS=${SUPPORTED_NODEJS_TESTS_VERSIONS:-10.20.1 11.15.0 12.17.0 13.14.0 14.3.0} + SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1} diff --git a/taskcluster/tc-build-utils.sh b/taskcluster/tc-build-utils.sh index 26234a4f..5530517e 100755 --- a/taskcluster/tc-build-utils.sh +++ b/taskcluster/tc-build-utils.sh @@ -110,7 +110,7 @@ do_deepspeech_nodejs_build() export PATH="${PYTHON27}:$PATH" - for node in ${SUPPORTED_NODEJS_VERSIONS}; do + for node in ${SUPPORTED_NODEJS_BUILD_VERSIONS}; do EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" make -C native_client/javascript \ TARGET=${SYSTEM_TARGET} \ RASPBIAN=${SYSTEM_RASPBIAN} \ From d24fb70869e90b25cc6a6c838e9d162d86328411 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 2 Jun 2020 14:56:26 +0200 Subject: [PATCH 14/17] Update node-gyp cache --- taskcluster/.shared.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 24c73887..672380c5 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -57,8 +57,8 @@ nodejs: prep_14: '/usr/bin/wget.exe https://nodejs.org/dist/v14.3.0/node-v14.3.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v14.3.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v14.3.0-win-x64/:$PATH' system: node_gyp_cache: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.3/artifacts/public/node-gyp-cache.tar.gz' - namespace: 'project.deepspeech.node-gyp-cache.3' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.4/artifacts/public/node-gyp-cache.tar.gz' + namespace: 'project.deepspeech.node-gyp-cache.4' homebrew_builds: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz' namespace: 'project.deepspeech.homebrew_builds.5' From f5d4f7f9d6c904e4be6e2ec00c75607dfd7fad25 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Wed, 3 Jun 2020 11:52:32 +0200 Subject: [PATCH 15/17] Fix typo in description for NodeJS / ARMbian --- taskcluster/test-nodejs_10x-armbian-arm64-opt.yml | 2 +- taskcluster/test-nodejs_11x-armbian-arm64-opt.yml | 2 +- taskcluster/test-nodejs_12x-armbian-arm64-opt.yml | 2 +- taskcluster/test-nodejs_13x-armbian-arm64-opt.yml | 2 +- taskcluster/test-nodejs_13x_multiarchpkg-armbian-arm64-opt.yml | 2 +- taskcluster/test-nodejs_14x-armbian-arm64-opt.yml | 2 +- taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/taskcluster/test-nodejs_10x-armbian-arm64-opt.yml b/taskcluster/test-nodejs_10x-armbian-arm64-opt.yml index ee0e025c..8c3cf5ac 100644 --- a/taskcluster/test-nodejs_10x-armbian-arm64-opt.yml +++ b/taskcluster/test-nodejs_10x-armbian-arm64-opt.yml @@ -11,4 +11,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 10.x 16k" metadata: name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 10.x tests" - description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v10.x, CPU only, optimized version" + description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v10.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_11x-armbian-arm64-opt.yml b/taskcluster/test-nodejs_11x-armbian-arm64-opt.yml index e08c8773..f7af3d3b 100644 --- a/taskcluster/test-nodejs_11x-armbian-arm64-opt.yml +++ b/taskcluster/test-nodejs_11x-armbian-arm64-opt.yml @@ -11,4 +11,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 11.x 16k" metadata: name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 11.x tests" - description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v11.x, CPU only, optimized version" + description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v11.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_12x-armbian-arm64-opt.yml b/taskcluster/test-nodejs_12x-armbian-arm64-opt.yml index a83b2bb1..db647ce0 100644 --- a/taskcluster/test-nodejs_12x-armbian-arm64-opt.yml +++ b/taskcluster/test-nodejs_12x-armbian-arm64-opt.yml @@ -11,4 +11,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 12.x 16k" metadata: name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 12.x tests" - description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v12.x, CPU only, optimized version" + description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v12.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_13x-armbian-arm64-opt.yml b/taskcluster/test-nodejs_13x-armbian-arm64-opt.yml index 7681bb7a..cadffe01 100644 --- a/taskcluster/test-nodejs_13x-armbian-arm64-opt.yml +++ b/taskcluster/test-nodejs_13x-armbian-arm64-opt.yml @@ -11,4 +11,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k" metadata: name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 13.x tests" - description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v13.x, CPU only, optimized version" + description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v13.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_13x_multiarchpkg-armbian-arm64-opt.yml b/taskcluster/test-nodejs_13x_multiarchpkg-armbian-arm64-opt.yml index c62a40ca..47f01a89 100644 --- a/taskcluster/test-nodejs_13x_multiarchpkg-armbian-arm64-opt.yml +++ b/taskcluster/test-nodejs_13x_multiarchpkg-armbian-arm64-opt.yml @@ -11,4 +11,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k" metadata: name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS MultiArch Package 13.x tests" - description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS MultiArch Package v13.x, CPU only, optimized version" + description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS MultiArch Package v13.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x-armbian-arm64-opt.yml b/taskcluster/test-nodejs_14x-armbian-arm64-opt.yml index d576c7ca..db09c287 100644 --- a/taskcluster/test-nodejs_14x-armbian-arm64-opt.yml +++ b/taskcluster/test-nodejs_14x-armbian-arm64-opt.yml @@ -11,4 +11,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" metadata: name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 14.x tests" - description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v14.x, CPU only, optimized version" + description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v14.x, CPU only, optimized version" diff --git a/taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml b/taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml index 2f7b792e..6debe667 100644 --- a/taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml +++ b/taskcluster/test-nodejs_14x_multiarchpkg-armbian-arm64-opt.yml @@ -11,4 +11,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k" metadata: name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS MultiArch Package 14.x tests" - description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS MultiArch Package v14.x, CPU only, optimized version" + description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS MultiArch Package v14.x, CPU only, optimized version" From 75a320e87b094fff3cd19adb31d3673694d42160 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Wed, 3 Jun 2020 11:53:12 +0200 Subject: [PATCH 16/17] Enable ElectronJS / TFLite / Windows tests --- taskcluster/tc-electron_tflite-tests.sh | 71 +++++++++++++++++++ ...onjs_v8.0_multiarchpkg-win-tflite-opt.yml} | 2 +- ...onjs_v9.0_multiarchpkg-win-tflite-opt.yml} | 2 +- 3 files changed, 73 insertions(+), 2 deletions(-) create mode 100755 taskcluster/tc-electron_tflite-tests.sh rename taskcluster/{test-electronjs_v8.0_multiarchpkg-win-tflite-opt.yml.disabled => test-electronjs_v8.0_multiarchpkg-win-tflite-opt.yml} (92%) rename taskcluster/{test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml.disabled => test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml} (92%) diff --git a/taskcluster/tc-electron_tflite-tests.sh b/taskcluster/tc-electron_tflite-tests.sh new file mode 100755 index 00000000..f740a5bd --- /dev/null +++ b/taskcluster/tc-electron_tflite-tests.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +nodever=$1 +electronver=$2 + +if [ -z "${nodever}" ]; then + echo "No node version given, aborting." + exit 1 +fi; + +if [ -z "${electronver}" ]; then + echo "No electron version given, aborting." + exit 1 +fi; + +bitrate=$3 +set_ldc_sample_filename "${bitrate}" + +model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite} +model_name=$(basename "${model_source}") +model_name_mmap=$(basename "${model_source}") + +download_data + +node --version +npm --version + +NODE_ROOT="${DS_ROOT_TASK}/ds-test/" +NODE_CACHE="${DS_ROOT_TASK}/ds-test.cache/" +export NODE_PATH="${NODE_ROOT}/node_modules/" +export PATH="${NODE_ROOT}:${NODE_PATH}/.bin/:${NODE_PATH}/electron/dist/:$PATH" + +npm install --prefix ${NODE_ROOT} --cache ${NODE_CACHE} electron@${electronver} + +deepspeech_npm_url=$(get_dep_npm_pkg_url) +npm install --prefix ${NODE_ROOT} --cache ${NODE_CACHE} ${deepspeech_npm_url} + +if [ "${OS}" = "Darwin" ]; then + ln -s Electron.app/Contents/MacOS/Electron "${NODE_ROOT}/node_modules/electron/dist/node" +else + ln -s electron "${NODE_ROOT}/node_modules/electron/dist/node" + if [ -f "${NODE_ROOT}/node_modules/electron/dist//chrome-sandbox" ]; then + export ELECTRON_DISABLE_SANDBOX=1 + fi; +fi + +find ${NODE_ROOT}/node_modules/electron/dist/ + +which electron +which node + +if [ "${OS}" = "Linux" ]; then + export DISPLAY=':99.0' + Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & + xvfb_process=$! +fi + +node --version + +check_runtime_electronjs + +run_electronjs_inference_tests + +if [ "${OS}" = "Linux" ]; then + sleep 1 + kill -9 ${xvfb_process} || true +fi diff --git a/taskcluster/test-electronjs_v8.0_multiarchpkg-win-tflite-opt.yml.disabled b/taskcluster/test-electronjs_v8.0_multiarchpkg-win-tflite-opt.yml similarity index 92% rename from taskcluster/test-electronjs_v8.0_multiarchpkg-win-tflite-opt.yml.disabled rename to taskcluster/test-electronjs_v8.0_multiarchpkg-win-tflite-opt.yml index ef2f693a..5d2531f0 100644 --- a/taskcluster/test-electronjs_v8.0_multiarchpkg-win-tflite-opt.yml.disabled +++ b/taskcluster/test-electronjs_v8.0_multiarchpkg-win-tflite-opt.yml @@ -8,7 +8,7 @@ build: > ${system.sox_win} && ${nodejs.win.prep_12} args: - tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 8.0.1 16k" + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron_tflite-tests.sh 12.x 8.0.1 16k" metadata: name: "DeepSpeech Windows AMD64 TFLite ElectronJS MultiArch Package v8.0 tests" description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v8.0, TFLite only, optimized version" diff --git a/taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml.disabled b/taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml similarity index 92% rename from taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml.disabled rename to taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml index 0813a7a0..4de742c3 100644 --- a/taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml.disabled +++ b/taskcluster/test-electronjs_v9.0_multiarchpkg-win-tflite-opt.yml @@ -8,7 +8,7 @@ build: > ${system.sox_win} && ${nodejs.win.prep_12} args: - tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k" + tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron_tflite-tests.sh 12.x 9.0.1 16k" metadata: name: "DeepSpeech Windows AMD64 TFLite ElectronJS MultiArch Package v9.0 tests" description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v9.0, TFLite only, optimized version" From 3ba3e10ecdad81a2b722f6d7db15585311f521a1 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Wed, 3 Jun 2020 17:05:59 +0200 Subject: [PATCH 17/17] Fix wrong nodejs version for Windows tests --- taskcluster/.shared.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 672380c5..dadb1094 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -52,7 +52,7 @@ nodejs: win: prep_10: '/usr/bin/wget.exe https://nodejs.org/dist/v10.20.1/node-v10.20.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v10.20.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v10.20.1-win-x64/:$PATH' prep_11: '/usr/bin/wget.exe https://nodejs.org/dist/v11.15.0/node-v11.15.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v11.15.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v11.15.0-win-x64/:$PATH' - prep_12: '/usr/bin/wget.exe https://nodejs.org/dist/v12.14.1/node-v12.14.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v12.14.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v12.14.1-win-x64/:$PATH' + prep_12: '/usr/bin/wget.exe https://nodejs.org/dist/v12.17.0/node-v12.17.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v12.17.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v12.17.0-win-x64/:$PATH' prep_13: '/usr/bin/wget.exe https://nodejs.org/dist/v13.14.0/node-v13.14.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v13.14.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v13.14.0-win-x64/:$PATH' prep_14: '/usr/bin/wget.exe https://nodejs.org/dist/v14.3.0/node-v14.3.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v14.3.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v14.3.0-win-x64/:$PATH' system: