Merge pull request #1871 from coqui-ai/training-tests

Training tests

.github/workflows/build-and-test.yml

@@ -607,6 +607,96 @@ jobs:
           bitrate: ${{ matrix.bitrate }}
           model-kind: ${{ matrix.models }}
         timeout-minutes: 5
+  training-basic-tests:
+    name: "Lin|Basic training tests"
+    runs-on: ubuntu-20.04
+    needs: [build-ctc-decoder-Linux]
+    strategy:
+      matrix:
+        bitrate: [8k, 16k]
+        pyver: [3.6, 3.7]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.pyver }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "coqui_stt_ctcdecoder-Linux-${{ matrix.pyver }}.whl"
+      - run: |
+          wget -O temp.zip https://github.com/coqui-ai/STT/releases/download/v0.9.3/convert_graphdef_memmapped_format.linux.amd64.zip
+          unzip temp.zip
+          rm temp.zip
+          chmod +x convert_graphdef_memmapped_format
+          mv convert_graphdef_memmapped_format /tmp
+      - run: |
+          mkdir -p ${CI_ARTIFACTS_DIR} || true
+      - name: Run basic training tests
+        run: |
+          python -m pip install coqui_stt_ctcdecoder-*.whl
+          ./ci_scripts/train-tests.sh ${{ matrix.bitrate }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}-output_graph.pb
+          path: ${{ github.workspace }}/artifacts/output_graph.pb
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}-output_graph.pbmm
+          path: ${{ github.workspace }}/artifacts/output_graph.pbmm
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}-output_graph.tflite
+          path: ${{ github.workspace }}/artifacts/output_graph.tflite
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}-checkpoint.tar.xz
+          path: ${{ github.workspace }}/artifacts/checkpoint.tar.xz
+  training-extra-tests:
+    name: "Lin|Extra training tests"
+    runs-on: ubuntu-20.04
+    needs: [build-ctc-decoder-Linux]
+    strategy:
+      matrix:
+        bitrate: [8k, 16k]
+        pyver: [3.6, 3.7]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.pyver }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "coqui_stt_ctcdecoder-Linux-${{ matrix.pyver }}.whl"
+      - run: |
+          wget -O temp.zip https://github.com/coqui-ai/STT/releases/download/v0.9.3/convert_graphdef_memmapped_format.linux.amd64.zip
+          unzip temp.zip
+          rm temp.zip
+          chmod +x convert_graphdef_memmapped_format
+          mv convert_graphdef_memmapped_format /tmp
+      - run: |
+          mkdir -p ${CI_ARTIFACTS_DIR} || true
+      - run: |
+          sudo apt-get install -y --no-install-recommends libopus0
+      - name: Run extra training tests
+        run: |
+          python -m pip install coqui_stt_ctcdecoder-*.whl
+          ./ci_scripts/train-extra-tests.sh ${{ matrix.bitrate }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}-output_graph.pb
+          path: ${{ github.workspace }}/artifacts/output_graph.pb
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}-output_graph.pbmm
+          path: ${{ github.workspace }}/artifacts/output_graph.pbmm
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}-output_graph.tflite
+          path: ${{ github.workspace }}/artifacts/output_graph.tflite
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}-checkpoint.tar.xz
+          path: ${{ github.workspace }}/artifacts/checkpoint.tar.xz
   # macOS jobs
   swig_macOS:
     name: "Mac|Build SWIG"

.github/workflows/lint.yml

@@ -0,0 +1,32 @@
+name: "Lints"
+on:
+  pull_request:
+defaults:
+  run:
+    shell: bash
+jobs:
+  training-unittests:
+    name: "Lin|Training unittests"
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        pyver: [3.6, 3.7]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.pyver }}
+      - name: Run training unittests
+        run: |
+          ./ci_scripts/train-unittests.sh
+  pre-commit-checks:
+    name: "Lin|Pre-commit checks"
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Run pre-comit checks
+        run: |
+          python .pre-commit-2.11.1.pyz run --all-files

bin/run-ci-ldc93s1_new_sdb_csv.sh

@@ -24,10 +24,10 @@ fi;
 export CUDA_VISIBLE_DEVICES=0
 
 python -u train.py --show_progressbar false --early_stop false \
-  --train_files ${ldc93s1_sdb},${ldc93s1_csv} --train_batch_size 1 \
+  --train_files ${ldc93s1_sdb} ${ldc93s1_csv} --train_batch_size 1 \
   --feature_cache '/tmp/ldc93s1_cache_sdb_csv' \
-  --dev_files ${ldc93s1_sdb},${ldc93s1_csv} --dev_batch_size 1 \
+  --dev_files ${ldc93s1_sdb} ${ldc93s1_csv} --dev_batch_size 1 \
-  --test_files ${ldc93s1_sdb},${ldc93s1_csv} --test_batch_size 1 \
+  --test_files ${ldc93s1_sdb} ${ldc93s1_csv} --test_batch_size 1 \
   --n_hidden 100 --epochs $epoch_count \
   --max_to_keep 1 --checkpoint_dir '/tmp/ckpt_sdb_csv' \
   --learning_rate 0.001 --dropout_rate 0.05  --export_dir '/tmp/train_sdb_csv' \

ci_scripts/train-extra-tests.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+
+bitrate=$1
+decoder_src=$2
+if [ "$decoder_src" != "--pypi" ]; then
+    # Use decoder package built in this CI group
+    export DS_NODECODER=1
+fi
+
+mkdir -p /tmp/train || true
+mkdir -p /tmp/train_tflite || true
+
+set -o pipefail
+python -m pip install --upgrade pip setuptools wheel | cat
+python -m pip install --upgrade . | cat
+set +o pipefail
+
+# Prepare correct arguments for training
+case "${bitrate}" in
+    8k)
+        sample_rate=8000
+        sample_name='LDC93S1_pcms16le_1_8000.wav'
+    ;;
+    16k)
+        sample_rate=16000
+        sample_name='LDC93S1_pcms16le_1_16000.wav'
+    ;;
+esac
+
+# Easier to rename to that we can exercize the LDC93S1 importer code to
+# generate the CSV file.
+echo "Moving ${sample_name} to LDC93S1.wav"
+mv "data/smoke_test/${sample_name}" "data/smoke_test/LDC93S1.wav"
+
+# Testing single SDB source
+time ./bin/run-ci-ldc93s1_new_sdb.sh 220 "${sample_rate}"
+# Testing interleaved source (SDB+CSV combination) - run twice to test preprocessed features
+time ./bin/run-ci-ldc93s1_new_sdb_csv.sh 109 "${sample_rate}"
+time ./bin/run-ci-ldc93s1_new_sdb_csv.sh 1 "${sample_rate}"
+
+# Test --metrics_files training argument
+time ./bin/run-ci-ldc93s1_new_metrics.sh 2 "${sample_rate}"
+
+# Test training with bytes output mode
+time ./bin/run-ci-ldc93s1_new_bytes.sh 200 "${sample_rate}"
+time ./bin/run-ci-ldc93s1_new_bytes_tflite.sh "${sample_rate}"
+
+tar -cf - \
+    -C /tmp/ckpt/ . \
+    | ${XZ} > ${CI_ARTIFACTS_DIR}/checkpoint.tar.xz
+
+# Save exported model artifacts from bytes output mode training
+cp /tmp/train_bytes/output_graph.pb ${CI_ARTIFACTS_DIR}/output_graph.pb
+cp /tmp/train_bytes_tflite/output_graph.tflite ${CI_ARTIFACTS_DIR}/output_graph.tflite
+
+/tmp/convert_graphdef_memmapped_format --in_graph=/tmp/train_bytes/output_graph.pb --out_graph=/tmp/train_bytes/output_graph.pbmm
+cp /tmp/train_bytes/output_graph.pbmm ${CI_ARTIFACTS_DIR}
+
+# Test resuming from checkpoints created above
+# SDB, resuming from checkpoint
+time ./bin/run-ci-ldc93s1_checkpoint_sdb.sh
+
+# Bytes output mode, resuming from checkpoint
+time ./bin/run-ci-ldc93s1_checkpoint_bytes.sh

ci_scripts/train-tests.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+
+bitrate=$1
+decoder_src=$2
+if [ "$decoder_src" != "--pypi" ]; then
+    # Use decoder package built in this CI group
+    export DS_NODECODER=1
+fi
+
+mkdir -p /tmp/train || true
+mkdir -p /tmp/train_tflite || true
+
+set -o pipefail
+python -m pip install --upgrade pip setuptools wheel | cat
+python -m pip install --upgrade . | cat
+set +o pipefail
+
+# Prepare correct arguments for training
+case "${bitrate}" in
+    8k)
+        sample_rate=8000
+        sample_name='LDC93S1_pcms16le_1_8000.wav'
+    ;;
+    16k)
+        sample_rate=16000
+        sample_name='LDC93S1_pcms16le_1_16000.wav'
+    ;;
+esac
+
+# Easier to rename to that we can exercize the LDC93S1 importer code to
+# generate the CSV file.
+echo "Moving ${sample_name} to LDC93S1.wav"
+mv "data/smoke_test/${sample_name}" "data/smoke_test/LDC93S1.wav"
+
+# Run twice to test preprocessed features
+time ./bin/run-ci-ldc93s1_new.sh 249 "${sample_rate}"
+time ./bin/run-ci-ldc93s1_new.sh 1 "${sample_rate}"
+time ./bin/run-ci-ldc93s1_tflite.sh "${sample_rate}"
+
+tar -cf - \
+    -C /tmp/ckpt/ . \
+    | ${XZ} > ${CI_ARTIFACTS_DIR}/checkpoint.tar.xz
+
+cp /tmp/train/output_graph.pb ${CI_ARTIFACTS_DIR}
+cp /tmp/train_tflite/output_graph.tflite ${CI_ARTIFACTS_DIR}
+
+/tmp/convert_graphdef_memmapped_format --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm
+cp /tmp/train/output_graph.pbmm ${CI_ARTIFACTS_DIR}
+
+time ./bin/run-ci-ldc93s1_checkpoint.sh

ci_scripts/train-unittests.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -xe
+
+source $(dirname "$0")/all-vars.sh
+source $(dirname "$0")/all-utils.sh
+
+set -o pipefail
+pip install --upgrade pip setuptools wheel | cat
+pip install --upgrade . | cat
+set +o pipefail
+
+python -m unittest

training/coqui_stt_training/util/config.py

@@ -466,7 +466,7 @@ class _SttConfig(Coqpit):
     )
 
     # Decoder
-    bytes_output_mode: float = field(
+    bytes_output_mode: bool = field(
         default=False,
         metadata=dict(
             help="enable Bytes Output Mode mode. When this is used the model outputs UTF-8 byte values directly rather than using an alphabet mapping. The --alphabet_config_path option will be ignored. See the training documentation for more details."