From 0b78f4ff0130c05f783bd12d343b34192489f43c Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Fri, 29 May 2020 17:06:08 +0530 Subject: [PATCH 1/5] move unittest to TC --- taskcluster/tc-train-unittests.sh | 24 +++++++++++++++++++ ...ing-unittests_8k-linux-amd64-py36m-opt.yml | 13 ++++++++++ 2 files changed, 37 insertions(+) create mode 100644 taskcluster/tc-train-unittests.sh create mode 100644 taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml diff --git a/taskcluster/tc-train-unittests.sh b/taskcluster/tc-train-unittests.sh new file mode 100644 index 00000000..dbae276e --- /dev/null +++ b/taskcluster/tc-train-unittests.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" + +bitrate=$2 + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true +mkdir -p /tmp/train || true +mkdir -p /tmp/train_tflite || true + +virtualenv_activate "${pyalias}" "deepspeech" + +set -o pipefail +pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat +pushd ${HOME}/DeepSpeech/ds + pip install --upgrade . | cat +popd +set +o pipefail + +python -m unittest diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml new file mode 100644 index 00000000..d507fc7e --- /dev/null +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml @@ -0,0 +1,13 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + system_setup: + > + apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m 8k" + metadata: + name: "DeepSpeech Linux AMD64 CPU 8kHz training Py3.6" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 8kHz Python 3.6, CPU only, optimized version" + \ No newline at end of file From ccca1c1fed17e5586cce43a6519d0914761c1e2c Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Fri, 29 May 2020 20:28:52 +0530 Subject: [PATCH 2/5] add tests to TC and update travis --- .travis.yml | 8 -------- taskcluster/tc-train-unittests.sh | 9 +-------- ...t-training-unittests_8k-linux-amd64-py35m-opt.yml | 12 ++++++++++++ ...t-training-unittests_8k-linux-amd64-py36m-opt.yml | 8 ++++---- ...t-training-unittests_8k-linux-amd64-py37m-opt.yml | 12 ++++++++++++ 5 files changed, 29 insertions(+), 20 deletions(-) create mode 100644 taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml create mode 100644 taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml diff --git a/.travis.yml b/.travis.yml index ca33852f..f64b1a34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,11 +20,3 @@ jobs: fi cardboardlinter --refspec $TRAVIS_BRANCH -n auto; fi - - name: python unit tests - install: - - pip install --upgrade -r requirements_tests.txt; - pip install --upgrade . - script: - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - python -m unittest; - fi diff --git a/taskcluster/tc-train-unittests.sh b/taskcluster/tc-train-unittests.sh index dbae276e..4ed6931c 100644 --- a/taskcluster/tc-train-unittests.sh +++ b/taskcluster/tc-train-unittests.sh @@ -4,13 +4,7 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" - -bitrate=$2 - mkdir -p ${TASKCLUSTER_ARTIFACTS} || true -mkdir -p /tmp/train || true -mkdir -p /tmp/train_tflite || true virtualenv_activate "${pyalias}" "deepspeech" @@ -18,7 +12,6 @@ set -o pipefail pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 | cat pushd ${HOME}/DeepSpeech/ds pip install --upgrade . | cat + python -m unittest popd set +o pipefail - -python -m unittest diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml new file mode 100644 index 00000000..81490ec0 --- /dev/null +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml @@ -0,0 +1,12 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + system_setup: + > + apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.5.8:m" + metadata: + name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.5" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.5, CPU only, optimized version" diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml index d507fc7e..8ca12a88 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml @@ -6,8 +6,8 @@ build: > apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} args: - tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m 8k" + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m" metadata: - name: "DeepSpeech Linux AMD64 CPU 8kHz training Py3.6" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 8kHz Python 3.6, CPU only, optimized version" - \ No newline at end of file + name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.6" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.6, CPU only, optimized version" + \ No newline at end of file diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml new file mode 100644 index 00000000..3b34eadc --- /dev/null +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml @@ -0,0 +1,12 @@ +build: + template_file: test-linux-opt-base.tyml + dependencies: + - "linux-amd64-ctc-opt" + system_setup: + > + apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + args: + tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.7.6:m" + metadata: + name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.7" + description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.7, CPU only, optimized version" From 73c4f3a201eae25e3ed5eeb892174d01f791b6c7 Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Fri, 29 May 2020 20:50:36 +0530 Subject: [PATCH 3/5] update tc-train-unittest.sh --- taskcluster/tc-train-unittests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskcluster/tc-train-unittests.sh b/taskcluster/tc-train-unittests.sh index 4ed6931c..4be52e90 100644 --- a/taskcluster/tc-train-unittests.sh +++ b/taskcluster/tc-train-unittests.sh @@ -4,6 +4,8 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh +extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias" + mkdir -p ${TASKCLUSTER_ARTIFACTS} || true virtualenv_activate "${pyalias}" "deepspeech" From 84d2f2a5f13ce8e7c3ae0f11b8fbe8503102ca30 Mon Sep 17 00:00:00 2001 From: Shubham Kumar Date: Fri, 29 May 2020 20:54:27 +0530 Subject: [PATCH 4/5] update description of test-training-unittests*yml --- .../test-training-unittests_8k-linux-amd64-py35m-opt.yml | 2 +- .../test-training-unittests_8k-linux-amd64-py36m-opt.yml | 2 +- .../test-training-unittests_8k-linux-amd64-py37m-opt.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml index 81490ec0..d33ad21d 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml @@ -9,4 +9,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.5.8:m" metadata: name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.5" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.5, CPU only, optimized version" + description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.5, for CPU only, and optimized version" diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml index 8ca12a88..195ddfe9 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml @@ -9,5 +9,5 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m" metadata: name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.6" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.6, CPU only, optimized version" + description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.6, for CPU only, and optimized version" \ No newline at end of file diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml index 3b34eadc..1f9aabc3 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml @@ -9,4 +9,4 @@ build: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.7.6:m" metadata: name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.7" - description: "Training a DeepSpeech LDC93S1 model for Linux/AMD64 Python 3.7, CPU only, optimized version" + description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.7, for CPU only, and optimized version" From ab2ba41c7fc79ebac02ed17abb8ce578a8d62da6 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Sat, 30 May 2020 14:45:17 +0200 Subject: [PATCH 5/5] Convert path to str to fix Python 3.5 compat --- training/deepspeech_training/util/importers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/training/deepspeech_training/util/importers.py b/training/deepspeech_training/util/importers.py index 4868e4a3..a4c3c326 100644 --- a/training/deepspeech_training/util/importers.py +++ b/training/deepspeech_training/util/importers.py @@ -42,15 +42,17 @@ def get_validate_label(args): :return: The user-supplied validate_label function :type: function """ + # Python 3.5 does not support passing a pathlib.Path to os.path.* methods if 'validate_label_locale' not in args or (args.validate_label_locale is None): print('WARNING: No --validate_label_locale specified, your might end with inconsistent dataset.') return validate_label_eng - if not os.path.exists(os.path.abspath(args.validate_label_locale)): + validate_label_locale = str(args.validate_label_locale) + if not os.path.exists(os.path.abspath(validate_label_locale)): print('ERROR: Inexistent --validate_label_locale specified. Please check.') return None - module_dir = os.path.abspath(os.path.dirname(args.validate_label_locale)) + module_dir = os.path.abspath(os.path.dirname(validate_label_locale)) sys.path.insert(1, module_dir) - fname = os.path.basename(args.validate_label_locale).replace('.py', '') + fname = os.path.basename(validate_label_locale).replace('.py', '') locale_module = importlib.import_module(fname, package=None) return locale_module.validate_label