Merge remote-tracking branch 'upstream/master' into r0.8-update
This commit is contained in:
commit
797cc2181c
15
.gitignore
vendored
15
.gitignore
vendored
@ -14,14 +14,21 @@
|
||||
/native_client/build
|
||||
/native_client/*.egg-info
|
||||
/native_client/dist
|
||||
/native_client/deepspeech
|
||||
/native_client/ds-swig
|
||||
/native_client/libdeepspeech.so
|
||||
/native_client/node_modules
|
||||
/native_client/python/model.py
|
||||
/native_client/python/utils.py
|
||||
/native_client/python/model_wrap.cpp
|
||||
/native_client/python/utils_wrap.cpp
|
||||
/native_client/javascript/build
|
||||
/native_client/javascript/lib
|
||||
/native_client/javascript/package.json
|
||||
/native_client/javascript/package-lock.json
|
||||
/native_client/javascript/client.js
|
||||
/native_client/javascript/deepspeech_wrap.cxx
|
||||
/native_client/javascript/node_modules
|
||||
/native_client/python/MANIFEST.in
|
||||
/native_client/python/dist
|
||||
/native_client/python/impl.py
|
||||
/native_client/python/impl_wrap.cpp
|
||||
/doc/.build/
|
||||
/doc/xml-c/
|
||||
/doc/xml-java/
|
||||
|
@ -20,11 +20,3 @@ jobs:
|
||||
fi
|
||||
cardboardlinter --refspec $TRAVIS_BRANCH -n auto;
|
||||
fi
|
||||
- name: python unit tests
|
||||
install:
|
||||
- pip install --upgrade -r requirements_tests.txt;
|
||||
pip install --upgrade .
|
||||
script:
|
||||
- if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
|
||||
python -m unittest;
|
||||
fi
|
||||
|
@ -7,6 +7,7 @@ DeepSpeech.py
|
||||
Use "python3 import_cv2.py -h" for help
|
||||
"""
|
||||
import csv
|
||||
import itertools
|
||||
import os
|
||||
import subprocess
|
||||
import unicodedata
|
||||
@ -30,19 +31,20 @@ SAMPLE_RATE = 16000
|
||||
MAX_SECS = 10
|
||||
|
||||
|
||||
def _preprocess_data(tsv_dir, audio_dir, space_after_every_character=False):
|
||||
def _preprocess_data(tsv_dir, audio_dir, filter_obj, space_after_every_character=False):
|
||||
exclude = []
|
||||
for dataset in ["test", "dev", "train", "validated", "other"]:
|
||||
set_samples = _maybe_convert_set(dataset, tsv_dir, audio_dir, space_after_every_character)
|
||||
set_samples = _maybe_convert_set(dataset, tsv_dir, audio_dir, filter_obj, space_after_every_character)
|
||||
if dataset in ["test", "dev"]:
|
||||
exclude += set_samples
|
||||
if dataset == "validated":
|
||||
_maybe_convert_set("train-all", tsv_dir, audio_dir, space_after_every_character,
|
||||
_maybe_convert_set("train-all", tsv_dir, audio_dir, filter_obj, space_after_every_character,
|
||||
rows=set_samples, exclude=exclude)
|
||||
|
||||
|
||||
def one_sample(sample):
|
||||
""" Take a audio file, and optionally convert it to 16kHz WAV """
|
||||
def one_sample(args):
|
||||
""" Take an audio file, and optionally convert it to 16kHz WAV """
|
||||
sample, filter_obj = args
|
||||
mp3_filename = sample[0]
|
||||
if not os.path.splitext(mp3_filename.lower())[1] == ".mp3":
|
||||
mp3_filename += ".mp3"
|
||||
@ -58,7 +60,7 @@ def one_sample(sample):
|
||||
["soxi", "-s", wav_filename], stderr=subprocess.STDOUT
|
||||
)
|
||||
)
|
||||
label = label_filter_fun(sample[1])
|
||||
label = filter_obj.filter(sample[1])
|
||||
rows = []
|
||||
counter = get_counter()
|
||||
if file_size == -1:
|
||||
@ -82,7 +84,7 @@ def one_sample(sample):
|
||||
return (counter, rows)
|
||||
|
||||
|
||||
def _maybe_convert_set(dataset, tsv_dir, audio_dir, space_after_every_character=None, rows=None, exclude=None):
|
||||
def _maybe_convert_set(dataset, tsv_dir, audio_dir, filter_obj, space_after_every_character=None, rows=None, exclude=None):
|
||||
exclude_transcripts = set()
|
||||
exclude_speakers = set()
|
||||
if exclude is not None:
|
||||
@ -109,7 +111,8 @@ def _maybe_convert_set(dataset, tsv_dir, audio_dir, space_after_every_character=
|
||||
print("Importing mp3 files...")
|
||||
pool = Pool()
|
||||
bar = progressbar.ProgressBar(max_value=num_samples, widgets=SIMPLE_BAR)
|
||||
for i, processed in enumerate(pool.imap_unordered(one_sample, samples), start=1):
|
||||
samples_with_context = itertools.zip_longest(samples, [], fillvalue=filter_obj)
|
||||
for i, processed in enumerate(pool.imap_unordered(one_sample, samples_with_context), start=1):
|
||||
counter += processed[0]
|
||||
rows += processed[1]
|
||||
bar.update(i)
|
||||
@ -160,50 +163,60 @@ def _maybe_convert_wav(mp3_filename, wav_filename):
|
||||
except sox.core.SoxError:
|
||||
pass
|
||||
|
||||
class LabelFilter:
|
||||
def __init__(self, normalize, alphabet, validate_fun):
|
||||
self.normalize = normalize
|
||||
self.alphabet = alphabet
|
||||
self.validate_fun = validate_fun
|
||||
|
||||
if __name__ == "__main__":
|
||||
PARSER = get_importers_parser(description="Import CommonVoice v2.0 corpora")
|
||||
PARSER.add_argument("tsv_dir", help="Directory containing tsv files")
|
||||
PARSER.add_argument(
|
||||
"--audio_dir",
|
||||
help='Directory containing the audio clips - defaults to "<tsv_dir>/clips"',
|
||||
)
|
||||
PARSER.add_argument(
|
||||
"--filter_alphabet",
|
||||
help="Exclude samples with characters not in provided alphabet",
|
||||
)
|
||||
PARSER.add_argument(
|
||||
"--normalize",
|
||||
action="store_true",
|
||||
help="Converts diacritic characters to their base ones",
|
||||
)
|
||||
PARSER.add_argument(
|
||||
"--space_after_every_character",
|
||||
action="store_true",
|
||||
help="To help transcript join by white space",
|
||||
)
|
||||
|
||||
PARAMS = PARSER.parse_args()
|
||||
validate_label = get_validate_label(PARAMS)
|
||||
|
||||
AUDIO_DIR = (
|
||||
PARAMS.audio_dir if PARAMS.audio_dir else os.path.join(PARAMS.tsv_dir, "clips")
|
||||
)
|
||||
ALPHABET = Alphabet(PARAMS.filter_alphabet) if PARAMS.filter_alphabet else None
|
||||
|
||||
def label_filter_fun(label):
|
||||
if PARAMS.normalize:
|
||||
def filter(self, label):
|
||||
if self.normalize:
|
||||
label = (
|
||||
unicodedata.normalize("NFKD", label.strip())
|
||||
.encode("ascii", "ignore")
|
||||
.decode("ascii", "ignore")
|
||||
)
|
||||
label = validate_label(label)
|
||||
if ALPHABET and label:
|
||||
label = self.validate_fun(label)
|
||||
if self.alphabet and label:
|
||||
try:
|
||||
ALPHABET.encode(label)
|
||||
self.alphabet.encode(label)
|
||||
except KeyError:
|
||||
label = None
|
||||
return label
|
||||
|
||||
_preprocess_data(PARAMS.tsv_dir, AUDIO_DIR, PARAMS.space_after_every_character)
|
||||
def main():
|
||||
parser = get_importers_parser(description="Import CommonVoice v2.0 corpora")
|
||||
parser.add_argument("tsv_dir", help="Directory containing tsv files")
|
||||
parser.add_argument(
|
||||
"--audio_dir",
|
||||
help='Directory containing the audio clips - defaults to "<tsv_dir>/clips"',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--filter_alphabet",
|
||||
help="Exclude samples with characters not in provided alphabet",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--normalize",
|
||||
action="store_true",
|
||||
help="Converts diacritic characters to their base ones",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--space_after_every_character",
|
||||
action="store_true",
|
||||
help="To help transcript join by white space",
|
||||
)
|
||||
|
||||
params = parser.parse_args()
|
||||
validate_label = get_validate_label(params)
|
||||
|
||||
audio_dir = (
|
||||
params.audio_dir if params.audio_dir else os.path.join(params.tsv_dir, "clips")
|
||||
)
|
||||
alphabet = Alphabet(params.filter_alphabet) if params.filter_alphabet else None
|
||||
|
||||
filter_obj = LabelFilter(params.normalize, alphabet, validate_label)
|
||||
_preprocess_data(params.tsv_dir, audio_dir, filter_obj,
|
||||
params.space_after_every_character)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
@ -32,7 +32,7 @@
|
||||
"host" : "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.v1.0.0-warpctc.arm/artifacts/public/"
|
||||
},
|
||||
"dependencies" : {
|
||||
"node-pre-gyp": "0.14.x",
|
||||
"node-pre-gyp": "0.15.x",
|
||||
"argparse": "1.0.x",
|
||||
"sox-stream": "2.0.x",
|
||||
"memory-stream": "0.0.3",
|
||||
|
@ -29,6 +29,7 @@ nodejs:
|
||||
prep_11: 'echo "deb http://deb.nodesource.com/node_11.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
|
||||
prep_12: 'echo "deb http://deb.nodesource.com/node_12.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
|
||||
prep_13: 'echo "deb http://deb.nodesource.com/node_13.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
|
||||
prep_14: 'echo "deb http://deb.nodesource.com/node_14.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
|
||||
packages_docs_bionic:
|
||||
apt: 'nodejs'
|
||||
apt_pinning: '(echo "Package: nodejs" && echo "Pin: origin deb.nodesource.com" && echo "Pin-Priority: 999") > /etc/apt/preferences'
|
||||
@ -41,26 +42,29 @@ nodejs:
|
||||
prep_11: 'echo "deb http://deb.nodesource.com/node_11.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
|
||||
prep_12: 'echo "deb http://deb.nodesource.com/node_12.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
|
||||
prep_13: 'echo "deb http://deb.nodesource.com/node_13.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
|
||||
prep_14: 'echo "deb http://deb.nodesource.com/node_14.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
|
||||
brew:
|
||||
prep_10: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 10.18.1'
|
||||
prep_10: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 10.20.1'
|
||||
prep_11: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 11.15.0'
|
||||
prep_12: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 12.8.1'
|
||||
prep_13: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 13.1.0'
|
||||
prep_12: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 12.17.0'
|
||||
prep_13: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 13.14.0'
|
||||
prep_14: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 14.3.0'
|
||||
win:
|
||||
prep_10: '/usr/bin/wget.exe https://nodejs.org/dist/v10.18.1/node-v10.18.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v10.18.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v10.18.1-win-x64/:$PATH'
|
||||
prep_10: '/usr/bin/wget.exe https://nodejs.org/dist/v10.20.1/node-v10.20.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v10.20.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v10.20.1-win-x64/:$PATH'
|
||||
prep_11: '/usr/bin/wget.exe https://nodejs.org/dist/v11.15.0/node-v11.15.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v11.15.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v11.15.0-win-x64/:$PATH'
|
||||
prep_12: '/usr/bin/wget.exe https://nodejs.org/dist/v12.14.1/node-v12.14.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v12.14.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v12.14.1-win-x64/:$PATH'
|
||||
prep_13: '/usr/bin/wget.exe https://nodejs.org/dist/v13.6.0/node-v13.6.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v13.6.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v13.6.0-win-x64/:$PATH'
|
||||
prep_12: '/usr/bin/wget.exe https://nodejs.org/dist/v12.17.0/node-v12.17.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v12.17.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v12.17.0-win-x64/:$PATH'
|
||||
prep_13: '/usr/bin/wget.exe https://nodejs.org/dist/v13.14.0/node-v13.14.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v13.14.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v13.14.0-win-x64/:$PATH'
|
||||
prep_14: '/usr/bin/wget.exe https://nodejs.org/dist/v14.3.0/node-v14.3.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v14.3.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v14.3.0-win-x64/:$PATH'
|
||||
system:
|
||||
node_gyp_cache:
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.2/artifacts/public/node-gyp-cache.tar.gz'
|
||||
namespace: 'project.deepspeech.node-gyp-cache.2'
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.4/artifacts/public/node-gyp-cache.tar.gz'
|
||||
namespace: 'project.deepspeech.node-gyp-cache.4'
|
||||
homebrew_builds:
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.4/artifacts/public/homebrew_builds.tar.gz'
|
||||
namespace: 'project.deepspeech.homebrew_builds.4'
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz'
|
||||
namespace: 'project.deepspeech.homebrew_builds.5'
|
||||
homebrew_tests:
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz'
|
||||
namespace: 'project.deepspeech.homebrew_tests.4'
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.6/artifacts/public/homebrew_tests.tar.gz'
|
||||
namespace: 'project.deepspeech.homebrew_tests.6'
|
||||
android_cache:
|
||||
arm64_v8a:
|
||||
android_24:
|
||||
|
@ -29,7 +29,7 @@ do_prepare_homebrew()
|
||||
export PATH=${_brew_instance}/bin:$PATH
|
||||
export HOMEBREW_LOGS="${_brew_instance}/homebrew.logs/"
|
||||
export HOMEBREW_CACHE="${_brew_instance}/homebrew.cache/"
|
||||
export BREW_FORMULAS_COMMIT=c3f06e4f17853bea8c35742923a9c43d3a244d35
|
||||
export BREW_FORMULAS_COMMIT=ddd39cf1b71452bfe9c5f17f45cc0118796b20d3
|
||||
|
||||
# Never fail on pre-existing homebrew/ directory
|
||||
mkdir -p "${_brew_instance}" || true
|
||||
@ -40,7 +40,7 @@ do_prepare_homebrew()
|
||||
curl -L ${BREW_URL} | tar xz --strip 1 -C "${_brew_instance}"
|
||||
fi;
|
||||
|
||||
check_homebrew
|
||||
check_homebrew "${_brew_instance}"
|
||||
|
||||
# Force an upgrade to fetch formulae
|
||||
brew search openssl
|
||||
@ -53,14 +53,16 @@ do_prepare_homebrew()
|
||||
|
||||
check_homebrew()
|
||||
{
|
||||
echo "local brew prefix ..."
|
||||
local_prefix=$(brew --prefix)
|
||||
echo "${local_prefix}"
|
||||
local _expected_prefix=$1
|
||||
|
||||
if [ "${BUILDS_BREW}" != "${local_prefix}" ]; then
|
||||
echo "local brew prefix ..."
|
||||
local _local_prefix=$(brew --prefix)
|
||||
echo "${_local_prefix}"
|
||||
|
||||
if [ "${_expected_prefix}" != "${_local_prefix}" ]; then
|
||||
echo "Weird state:"
|
||||
echo "BUILDS_BREW=${BUILDS_BREW}"
|
||||
echo "local_prefix=${local_prefix}"
|
||||
echo "_expected_prefix=${_expected_prefix}"
|
||||
echo "_local_prefix=${_local_prefix}"
|
||||
exit 1
|
||||
fi;
|
||||
}
|
||||
@ -90,7 +92,7 @@ prepare_homebrew_tests()
|
||||
|
||||
install_pkg_homebrew "nvm"
|
||||
source "${TESTS_BREW}/opt/nvm/nvm.sh"
|
||||
for node_ver in ${SUPPORTED_NODEJS_VERSIONS};
|
||||
for node_ver in ${SUPPORTED_NODEJS_TESTS_VERSIONS};
|
||||
do
|
||||
nvm install ${node_ver}
|
||||
done;
|
||||
|
@ -1,8 +1,8 @@
|
||||
build:
|
||||
template_file: homebrew-darwin-opt-base.tyml
|
||||
homebrew:
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.4/artifacts/public/homebrew_builds.tar.gz'
|
||||
namespace: 'project.deepspeech.homebrew_builds.4'
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.5/artifacts/public/homebrew_builds.tar.gz'
|
||||
namespace: 'project.deepspeech.homebrew_builds.5'
|
||||
scripts:
|
||||
build: "taskcluster/homebrew-build.sh --builds"
|
||||
package: "taskcluster/homebrew-package.sh --builds"
|
||||
|
@ -1,8 +1,8 @@
|
||||
build:
|
||||
template_file: homebrew-darwin-opt-base.tyml
|
||||
homebrew:
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz'
|
||||
namespace: 'project.deepspeech.homebrew_tests.4'
|
||||
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.6/artifacts/public/homebrew_tests.tar.gz'
|
||||
namespace: 'project.deepspeech.homebrew_tests.6'
|
||||
scripts:
|
||||
build: "taskcluster/homebrew-build.sh --tests"
|
||||
package: "taskcluster/homebrew-package.sh --tests"
|
||||
|
@ -16,7 +16,7 @@ mkdir -p ${devDir}
|
||||
|
||||
node-gyp list ${devDir}
|
||||
|
||||
for node in ${SUPPORTED_NODEJS_VERSIONS}; do
|
||||
for node in ${SUPPORTED_NODEJS_BUILD_VERSIONS}; do
|
||||
node-gyp install --devdir ${devDir} \
|
||||
--target=$node
|
||||
mkdir ${devDir}/${node}/x64/ || true
|
||||
|
@ -25,7 +25,7 @@ if [ "${OS}" = "Darwin" ]; then
|
||||
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root"
|
||||
|
||||
export HOMEBREW_NO_AUTO_UPDATE=1
|
||||
export BREW_URL=https://github.com/Homebrew/brew/tarball/2.1.14
|
||||
export BREW_URL=https://github.com/Homebrew/brew/tarball/2.2.17
|
||||
|
||||
export BUILDS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-builds"
|
||||
export TESTS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-tests"
|
||||
@ -87,5 +87,10 @@ SUPPORTED_PYTHON_VERSIONS=${SUPPORTED_PYTHON_VERSIONS:-3.5.8:ucs2 3.6.10:ucs2 3.
|
||||
# When updating NodeJS / ElectronJS supported versions, do not forget to increment
|
||||
# deepspeech.node-gyp-cache.<X> in both `system.node_gyp_cache` (taskcluster/.shared.yml)
|
||||
# and route index (taskcluster/node-gyp-cache.yml) to ensure the cache is updated
|
||||
SUPPORTED_NODEJS_VERSIONS=${SUPPORTED_NODEJS_VERSIONS:-10.18.1 11.15.0 12.8.1 13.1.0}
|
||||
SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1}
|
||||
#
|
||||
# Also, builds should always target first major branch release to ensure proper binary compatibility
|
||||
# and tests should always target latest major branch release version
|
||||
SUPPORTED_NODEJS_BUILD_VERSIONS=${SUPPORTED_NODEJS_BUILD_VERSIONS:-10.0.0 11.0.0 12.7.0 13.0.0 14.0.0}
|
||||
SUPPORTED_NODEJS_TESTS_VERSIONS=${SUPPORTED_NODEJS_TESTS_VERSIONS:-10.20.1 11.15.0 12.17.0 13.14.0 14.3.0}
|
||||
|
||||
SUPPORTED_ELECTRONJS_VERSIONS=${SUPPORTED_ELECTRONJS_VERSIONS:-5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1}
|
||||
|
@ -110,7 +110,7 @@ do_deepspeech_nodejs_build()
|
||||
|
||||
export PATH="${PYTHON27}:$PATH"
|
||||
|
||||
for node in ${SUPPORTED_NODEJS_VERSIONS}; do
|
||||
for node in ${SUPPORTED_NODEJS_BUILD_VERSIONS}; do
|
||||
EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" EXTRA_LDFLAGS="${EXTRA_LOCAL_LDFLAGS}" EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" make -C native_client/javascript \
|
||||
TARGET=${SYSTEM_TARGET} \
|
||||
RASPBIAN=${SYSTEM_RASPBIAN} \
|
||||
|
71
taskcluster/tc-electron_tflite-tests.sh
Executable file
71
taskcluster/tc-electron_tflite-tests.sh
Executable file
@ -0,0 +1,71 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
nodever=$1
|
||||
electronver=$2
|
||||
|
||||
if [ -z "${nodever}" ]; then
|
||||
echo "No node version given, aborting."
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
if [ -z "${electronver}" ]; then
|
||||
echo "No electron version given, aborting."
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
bitrate=$3
|
||||
set_ldc_sample_filename "${bitrate}"
|
||||
|
||||
model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite}
|
||||
model_name=$(basename "${model_source}")
|
||||
model_name_mmap=$(basename "${model_source}")
|
||||
|
||||
download_data
|
||||
|
||||
node --version
|
||||
npm --version
|
||||
|
||||
NODE_ROOT="${DS_ROOT_TASK}/ds-test/"
|
||||
NODE_CACHE="${DS_ROOT_TASK}/ds-test.cache/"
|
||||
export NODE_PATH="${NODE_ROOT}/node_modules/"
|
||||
export PATH="${NODE_ROOT}:${NODE_PATH}/.bin/:${NODE_PATH}/electron/dist/:$PATH"
|
||||
|
||||
npm install --prefix ${NODE_ROOT} --cache ${NODE_CACHE} electron@${electronver}
|
||||
|
||||
deepspeech_npm_url=$(get_dep_npm_pkg_url)
|
||||
npm install --prefix ${NODE_ROOT} --cache ${NODE_CACHE} ${deepspeech_npm_url}
|
||||
|
||||
if [ "${OS}" = "Darwin" ]; then
|
||||
ln -s Electron.app/Contents/MacOS/Electron "${NODE_ROOT}/node_modules/electron/dist/node"
|
||||
else
|
||||
ln -s electron "${NODE_ROOT}/node_modules/electron/dist/node"
|
||||
if [ -f "${NODE_ROOT}/node_modules/electron/dist//chrome-sandbox" ]; then
|
||||
export ELECTRON_DISABLE_SANDBOX=1
|
||||
fi;
|
||||
fi
|
||||
|
||||
find ${NODE_ROOT}/node_modules/electron/dist/
|
||||
|
||||
which electron
|
||||
which node
|
||||
|
||||
if [ "${OS}" = "Linux" ]; then
|
||||
export DISPLAY=':99.0'
|
||||
Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
|
||||
xvfb_process=$!
|
||||
fi
|
||||
|
||||
node --version
|
||||
|
||||
check_runtime_electronjs
|
||||
|
||||
run_electronjs_inference_tests
|
||||
|
||||
if [ "${OS}" = "Linux" ]; then
|
||||
sleep 1
|
||||
kill -9 ${xvfb_process} || true
|
||||
fi
|
19
taskcluster/tc-train-unittests.sh
Normal file
19
taskcluster/tc-train-unittests.sh
Normal file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
|
||||
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
||||
|
||||
virtualenv_activate "${pyalias}" "deepspeech"
|
||||
|
||||
set -o pipefail
|
||||
pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat
|
||||
pushd ${HOME}/DeepSpeech/ds
|
||||
pip install --upgrade . | cat
|
||||
python -m unittest
|
||||
popd
|
||||
set +o pipefail
|
@ -8,7 +8,7 @@ build:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_12}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 8.0.1 16k"
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron_tflite-tests.sh 12.x 8.0.1 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 TFLite ElectronJS MultiArch Package v8.0 tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v8.0, TFLite only, optimized version"
|
15
taskcluster/test-electronjs_v9.0-darwin-amd64-opt.yml
Normal file
15
taskcluster/test-electronjs_v9.0-darwin-amd64-opt.yml
Normal file
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-darwin-opt-base.tyml
|
||||
dependencies:
|
||||
- "darwin-amd64-cpu-opt"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
- "homebrew_tests-darwin-amd64"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.brew.prep_12}
|
||||
args:
|
||||
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech OSX AMD64 CPU ElectronJS v9.0 tests"
|
||||
description: "Testing DeepSpeech for OSX/AMD64 on ElectronJS v9.0, CPU only, optimized version"
|
14
taskcluster/test-electronjs_v9.0-win-amd64-opt.yml
Normal file
14
taskcluster/test-electronjs_v9.0-win-amd64-opt.yml
Normal file
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-win-opt-base.tyml
|
||||
dependencies:
|
||||
- "win-amd64-cpu-opt"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_12}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 CPU ElectronJS v9.0 tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS v9.0, CPU only, optimized version"
|
15
taskcluster/test-electronjs_v9.0_16k-linux-amd64-opt.yml
Normal file
15
taskcluster/test-electronjs_v9.0_16k-linux-amd64-opt.yml
Normal file
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "linux-amd64-cpu-opt"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} ${electronjs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU ElectronJS v9.0 tests (16kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on ElectronJS v9.0, CPU only, optimized version (16kHz)"
|
15
taskcluster/test-electronjs_v9.0_8k-linux-amd64-opt.yml
Normal file
15
taskcluster/test-electronjs_v9.0_8k-linux-amd64-opt.yml
Normal file
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "linux-amd64-cpu-opt"
|
||||
- "test-training_8k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt} ${electronjs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 8k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU ElectronJS v9.0 tests (8kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on ElectronJS v9.0, CPU only, optimized version (8kHz)"
|
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-win-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_12}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 CPU ElectronJS MultiArch Package v9.0 tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v9.0, CPU only, optimized version"
|
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-win-cuda-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-gpu"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_12}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 9.0.1 16k cuda"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 CUDA ElectronJS MultiArch Package v9.0 tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v9.0, CUDA, optimized version"
|
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-win-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-tflite"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_12}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-electron_tflite-tests.sh 12.x 9.0.1 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 TFLite ElectronJS MultiArch Package v9.0 tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on ElectronJS MultiArch Package v9.0, TFLite only, optimized version"
|
@ -11,4 +11,4 @@ build:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 10.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 10.x tests"
|
||||
description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v10.x, CPU only, optimized version"
|
||||
description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v10.x, CPU only, optimized version"
|
||||
|
@ -11,4 +11,4 @@ build:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 11.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 11.x tests"
|
||||
description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v11.x, CPU only, optimized version"
|
||||
description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v11.x, CPU only, optimized version"
|
||||
|
@ -11,4 +11,4 @@ build:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 12.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 12.x tests"
|
||||
description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v12.x, CPU only, optimized version"
|
||||
description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v12.x, CPU only, optimized version"
|
||||
|
@ -11,4 +11,4 @@ build:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 13.x tests"
|
||||
description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS v13.x, CPU only, optimized version"
|
||||
description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v13.x, CPU only, optimized version"
|
||||
|
@ -11,4 +11,4 @@ build:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS MultiArch Package 13.x tests"
|
||||
description: "Testing DeepSpeech forARMbian ARM64 Cortex-A53 on NodeJS MultiArch Package v13.x, CPU only, optimized version"
|
||||
description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS MultiArch Package v13.x, CPU only, optimized version"
|
||||
|
14
taskcluster/test-nodejs_14x-armbian-arm64-opt.yml
Normal file
14
taskcluster/test-nodejs_14x-armbian-arm64-opt.yml
Normal file
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-armbian-opt-base.tyml
|
||||
dependencies:
|
||||
- "linux-arm64-cpu-opt"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_buster.prep_14} && ${nodejs.packages_buster.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_buster.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS 14.x tests"
|
||||
description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS v14.x, CPU only, optimized version"
|
15
taskcluster/test-nodejs_14x-darwin-amd64-opt.yml
Normal file
15
taskcluster/test-nodejs_14x-darwin-amd64-opt.yml
Normal file
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-darwin-opt-base.tyml
|
||||
dependencies:
|
||||
- "darwin-amd64-cpu-opt"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
- "homebrew_tests-darwin-amd64"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.brew.prep_14}
|
||||
args:
|
||||
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech OSX AMD64 CPU NodeJS 14.x tests"
|
||||
description: "Testing DeepSpeech for OSX/AMD64 on NodeJS v14.x, CPU only, optimized version"
|
14
taskcluster/test-nodejs_14x-raspbian-rpi3-opt.yml
Normal file
14
taskcluster/test-nodejs_14x-raspbian-rpi3-opt.yml
Normal file
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-raspbian-opt-base.tyml
|
||||
dependencies:
|
||||
- "linux-rpi3-cpu-opt"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_buster.prep_14} && ${nodejs.packages_buster.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_buster.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS 14.x tests"
|
||||
description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS v14.x, CPU only, optimized version"
|
14
taskcluster/test-nodejs_14x-win-amd64-opt.yml
Normal file
14
taskcluster/test-nodejs_14x-win-amd64-opt.yml
Normal file
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-win-opt-base.tyml
|
||||
dependencies:
|
||||
- "win-amd64-cpu-opt"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_14}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 CPU NodeJS 14.x tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on NodeJS v14.x, CPU only, optimized version"
|
15
taskcluster/test-nodejs_14x_16k-linux-amd64-opt.yml
Normal file
15
taskcluster/test-nodejs_14x_16k-linux-amd64-opt.yml
Normal file
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "linux-amd64-cpu-opt"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU NodeJS 14.x tests (16kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x, CPU only, optimized version (16kHz)"
|
@ -0,0 +1,13 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "linux-amd64-cpu-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests-prod.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU NodeJS 14.x prod tests (16kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x on prod model, CPU only, optimized version (16kHz)"
|
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU NodeJS MultiArch Package 14.x tests (16kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x, CPU only, optimized version (16kHz)"
|
@ -0,0 +1,13 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests-prod.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU NodeJS MultiArch Package 14.x prod tests (16kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x on prod model, CPU only, optimized version (16kHz)"
|
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "node-package-tflite"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 TFLite NodeJS MultiArch Package 14.x tests (16kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x, TFLite only, optimized version (16kHz)"
|
@ -0,0 +1,13 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "linux-amd64-tflite-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 TFLite NodeJS 14.x prod tests (16kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x on prod model, TFLite, optimized version (16kHz)"
|
15
taskcluster/test-nodejs_14x_8k-linux-amd64-opt.yml
Normal file
15
taskcluster/test-nodejs_14x_8k-linux-amd64-opt.yml
Normal file
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "linux-amd64-cpu-opt"
|
||||
- "test-training_8k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 8k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU NodeJS 14.x tests (8kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x, CPU only, optimized version (8kHz)"
|
@ -0,0 +1,13 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "linux-amd64-cpu-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests-prod.sh 14.x 8k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU NodeJS 14.x prod tests (8kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x on prod model, CPU only, optimized version (8kHz)"
|
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
- "test-training_8k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 8k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU NodeJS MultiArch Package 14.x tests (8kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x, CPU only, optimized version (8kHz)"
|
@ -0,0 +1,13 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node-tests-prod.sh 14.x 8k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 CPU NodeJS MultiArch Package 14.x prod tests (8kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x on prod model, CPU only, optimized version (8kHz)"
|
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "node-package-tflite"
|
||||
- "test-training_8k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_8k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 8k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 TFLite NodeJS MultiArch Package 14.x tests (8kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS MultiArch Package v14.x, TFLite only, optimized version (8kHz)"
|
@ -0,0 +1,13 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
docker_image: "ubuntu:16.04"
|
||||
dependencies:
|
||||
- "linux-amd64-tflite-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_xenial.prep_14} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_xenial.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests-prod.sh 14.x 8k"
|
||||
metadata:
|
||||
name: "DeepSpeech Linux AMD64 TFLite NodeJS 14.x prod tests (8kHz)"
|
||||
description: "Testing DeepSpeech for Linux/AMD64 on NodeJS v14.x on prod model, TFLite, optimized version (8kHz)"
|
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-armbian-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_buster.prep_14} && ${nodejs.packages_buster.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_buster.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech ARMbian ARM64 Cortex-A53 CPU NodeJS MultiArch Package 14.x tests"
|
||||
description: "Testing DeepSpeech for ARMbian ARM64 Cortex-A53 on NodeJS MultiArch Package v14.x, CPU only, optimized version"
|
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-darwin-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
- "homebrew_tests-darwin-amd64"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.brew.prep_14}
|
||||
args:
|
||||
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech OSX AMD64 CPU NodeJS MultiArch Package 14.x tests"
|
||||
description: "Testing DeepSpeech for OSX/AMD64 on NodeJS MultiArch Package v14.x, CPU only, optimized version"
|
@ -0,0 +1,15 @@
|
||||
build:
|
||||
template_file: test-darwin-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-tflite"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
- "homebrew_tests-darwin-amd64"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.brew.prep_14}
|
||||
args:
|
||||
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech OSX AMD64 TFLite NodeJS MultiArch Package 14.x tests"
|
||||
description: "Testing DeepSpeech for OSX/AMD64 on NodeJS MultiArch Package v14.x, TFLite only, optimized version"
|
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-raspbian-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${nodejs.packages_buster.prep_14} && ${nodejs.packages_buster.apt_pinning} && apt-get -qq update && apt-get -qq -y install ${nodejs.packages_buster.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Raspbian RPi3/ARMv7 CPU NodeJS MultiArch Package 14.x tests"
|
||||
description: "Testing DeepSpeech for Raspbian RPi3/ARMv7 on NodeJS MultiArch Package v14.x, CPU only, optimized version"
|
14
taskcluster/test-nodejs_14x_multiarchpkg-win-amd64-opt.yml
Normal file
14
taskcluster/test-nodejs_14x_multiarchpkg-win-amd64-opt.yml
Normal file
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-win-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-cpu"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_14}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 CPU NodeJS MultiArch Package 14.x tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on NodeJS MultiArch Package v14.x, CPU only, optimized version"
|
14
taskcluster/test-nodejs_14x_multiarchpkg-win-cuda-opt.yml
Normal file
14
taskcluster/test-nodejs_14x_multiarchpkg-win-cuda-opt.yml
Normal file
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-win-cuda-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-gpu"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_14}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node-tests.sh 14.x 16k cuda"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 CUDA NodeJS MultiArch Package 14.x tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on NodeJS MultiArch Package v14.x, CUDA, optimized version"
|
14
taskcluster/test-nodejs_14x_multiarchpkg-win-tflite-opt.yml
Normal file
14
taskcluster/test-nodejs_14x_multiarchpkg-win-tflite-opt.yml
Normal file
@ -0,0 +1,14 @@
|
||||
build:
|
||||
template_file: test-win-opt-base.tyml
|
||||
dependencies:
|
||||
- "node-package-tflite"
|
||||
- "test-training_16k-linux-amd64-py36m-opt"
|
||||
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
|
||||
system_setup:
|
||||
>
|
||||
${system.sox_win} && ${nodejs.win.prep_14}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.win}/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 14.x 16k"
|
||||
metadata:
|
||||
name: "DeepSpeech Windows AMD64 TFLite NodeJS MultiArch Package 14.x tests"
|
||||
description: "Testing DeepSpeech for Windows/AMD64 on NodeJS MultiArch Package v14.x, TFLite only, optimized version"
|
@ -0,0 +1,12 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
dependencies:
|
||||
- "linux-amd64-ctc-opt"
|
||||
system_setup:
|
||||
>
|
||||
apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.5.8:m"
|
||||
metadata:
|
||||
name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.5"
|
||||
description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.5, for CPU only, and optimized version"
|
@ -0,0 +1,13 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
dependencies:
|
||||
- "linux-amd64-ctc-opt"
|
||||
system_setup:
|
||||
>
|
||||
apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m"
|
||||
metadata:
|
||||
name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.6"
|
||||
description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.6, for CPU only, and optimized version"
|
||||
|
@ -0,0 +1,12 @@
|
||||
build:
|
||||
template_file: test-linux-opt-base.tyml
|
||||
dependencies:
|
||||
- "linux-amd64-ctc-opt"
|
||||
system_setup:
|
||||
>
|
||||
apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt}
|
||||
args:
|
||||
tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.7.6:m"
|
||||
metadata:
|
||||
name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.7"
|
||||
description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.7, for CPU only, and optimized version"
|
@ -42,15 +42,17 @@ def get_validate_label(args):
|
||||
:return: The user-supplied validate_label function
|
||||
:type: function
|
||||
"""
|
||||
# Python 3.5 does not support passing a pathlib.Path to os.path.* methods
|
||||
if 'validate_label_locale' not in args or (args.validate_label_locale is None):
|
||||
print('WARNING: No --validate_label_locale specified, your might end with inconsistent dataset.')
|
||||
return validate_label_eng
|
||||
if not os.path.exists(os.path.abspath(args.validate_label_locale)):
|
||||
validate_label_locale = str(args.validate_label_locale)
|
||||
if not os.path.exists(os.path.abspath(validate_label_locale)):
|
||||
print('ERROR: Inexistent --validate_label_locale specified. Please check.')
|
||||
return None
|
||||
module_dir = os.path.abspath(os.path.dirname(args.validate_label_locale))
|
||||
module_dir = os.path.abspath(os.path.dirname(validate_label_locale))
|
||||
sys.path.insert(1, module_dir)
|
||||
fname = os.path.basename(args.validate_label_locale).replace('.py', '')
|
||||
fname = os.path.basename(validate_label_locale).replace('.py', '')
|
||||
locale_module = importlib.import_module(fname, package=None)
|
||||
return locale_module.validate_label
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user