Merge pull request #1855 from coqui-ai/linux-ci

Linux CI base

.github/actions/check_artifact_exists/main.js

@@ -8,51 +8,30 @@ const { throttling } = require('@octokit/plugin-throttling');
 const { GitHub } = require('@actions/github/lib/utils');
 
 async function getGoodArtifacts(client, owner, repo, name) {
-    const goodWorkflowArtifacts = await client.paginate(
-        "GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts",
-        {
-            owner: owner,
-            repo: repo,
-            run_id: github.context.runId,
-        },
-        (workflowArtifacts) => {
-            // console.log(" ==> workflowArtifacts", workflowArtifacts);
-            return workflowArtifacts.data.filter((a) => {
-                // console.log("==> Artifact check", a);
-                return a.name == name
-            })
-        }
-    );
-
-    console.log("==> maybe goodWorkflowArtifacts:", goodWorkflowArtifacts);
-    if (goodWorkflowArtifacts.length > 0) {
-        return goodWorkflowArtifacts;
-    }
-
     const goodRepoArtifacts = await client.paginate(
         "GET /repos/{owner}/{repo}/actions/artifacts",
         {
             owner: owner,
             repo: repo,
+            per_page: 100,
         },
-        (repoArtifacts) => {
+        (repoArtifacts, done) => {
             // console.log(" ==> repoArtifacts", repoArtifacts);
-            return repoArtifacts.data.filter((a) => {
+            const goodArtifacts = repoArtifacts.data.filter((a) => {
                 // console.log("==> Artifact check", a);
                 return a.name == name
-            })
+            });
+            if (goodArtifacts.length > 0) {
+                done();
+            }
+            return goodArtifacts;
         }
     );
 
     console.log("==> maybe goodRepoArtifacts:", goodRepoArtifacts);
-    if (goodRepoArtifacts.length > 0) {
     return goodRepoArtifacts;
 }
 
-    // We have not been able to find a repo artifact, it's really no good news
-    return [];
-}
-
 async function main() {
     const token = core.getInput("github_token", { required: true });
     const [owner, repo] = core.getInput("repo", { required: true }).split("/");

.github/actions/check_artifact_exists/package-lock.json

@@ -79,7 +79,8 @@
     "@octokit/openapi-types": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-6.0.0.tgz",
-      "integrity": "sha512-CnDdK7ivHkBtJYzWzZm7gEkanA7gKH6a09Eguz7flHw//GacPJLmkHA3f3N++MJmlxD1Fl+mB7B32EEpSCwztQ=="
+      "integrity": "sha512-CnDdK7ivHkBtJYzWzZm7gEkanA7gKH6a09Eguz7flHw//GacPJLmkHA3f3N++MJmlxD1Fl+mB7B32EEpSCwztQ==",
+      "dev": true
     },
     "@octokit/plugin-paginate-rest": {
       "version": "2.13.3",
@@ -104,6 +105,7 @@
       "version": "3.4.1",
       "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.4.1.tgz",
       "integrity": "sha512-qCQ+Z4AnL9OrXvV59EH3GzPxsB+WyqufoCjiCJXJxTbnt3W+leXbXw5vHrMp4NG9ltw00McFWIxIxNQAzLNoTA==",
+      "dev": true,
       "requires": {
         "@octokit/types": "^6.0.1",
         "bottleneck": "^2.15.3"
@@ -138,6 +140,7 @@
       "version": "6.13.0",
       "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.13.0.tgz",
       "integrity": "sha512-W2J9qlVIU11jMwKHUp5/rbVUeErqelCsO5vW5PKNb7wAXQVUz87Rc+imjlEvpvbH8yUb+KHmv8NEjVZdsdpyxA==",
+      "dev": true,
       "requires": {
         "@octokit/openapi-types": "^6.0.0"
       }
@@ -163,7 +166,8 @@
     "bottleneck": {
       "version": "2.19.5",
       "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz",
-      "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="
+      "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==",
+      "dev": true
     },
     "deprecation": {
       "version": "2.3.1",

.github/workflows/build-and-test.yml

@@ -25,33 +25,586 @@ jobs:
     name: "Lin|Build SWIG for Windows"
     runs-on: ubuntu-20.04
     env:
-      CFLAGS: "-static-libgcc -static-libstdc++"
+      swig_hash: "90cdbee6a69d13b39d734083b9f91069533b0d7b"
-      CXXFLAGS: "-static-libgcc -static-libstdc++"
     steps:
-      - run: |
-          sudo apt-get install -y --no-install-recommends autoconf automake bison build-essential mingw-w64
       - uses: actions/checkout@v2
         with:
-          repository: "lissyx/swig"
+          repository: "swig/swig"
-          ref: "fec7d5d3179833e37759ffc6532f86344982e26a"
+          ref: ${{ env.swig_hash }}
       - run: |
           mkdir -p build-static/
+      - uses: actions/cache@v2
+        id: swig-build-cache
+        with:
+          path: build-static/
+          key: swig-win-3-${{ env.swig_hash }}
+      - run: |
+          sudo apt-get install -y --no-install-recommends autoconf automake bison build-essential mingw-w64
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
       - run: |
           curl -sSL https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz > pcre-8.43.tar.gz
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
+      - run: |
           ./Tools/pcre-build.sh --host=x86_64-w64-mingw32
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
       - run: |
           sh autogen.sh
-          ./configure --host=x86_64-w64-mingw32 \
+          CFLAGS="-static-libgcc -static-libstdc++" \
+          CXXFLAGS="-static-libgcc -static-libstdc++" \
+          ./configure \
+            --host=x86_64-w64-mingw32 \
             --prefix=`pwd`/build-static/ \
             --program-prefix=ds-
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
       - run: |
           make -j
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
       - run: |
           make install
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
       - uses: actions/upload-artifact@v2
         with:
           name: ${{ github.job }}
           path: ${{ github.workspace }}/build-static/
+  swig_Linux:
+    name: "Lin|Build SWIG"
+    runs-on: ubuntu-20.04
+    env:
+      swig_hash: "90cdbee6a69d13b39d734083b9f91069533b0d7b"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          repository: "swig/swig"
+          ref: ${{ env.swig_hash }}
+      - run: |
+          mkdir -p build-static/
+      - uses: actions/cache@v2
+        id: swig-build-cache
+        with:
+          path: build-static/
+          key: swig-2-${{ runner.os }}-${{ env.swig_hash }}
+      - run: |
+          sudo apt-get install -y --no-install-recommends autoconf automake bison build-essential
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
+      - run: |
+          curl -sSL https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz > pcre-8.43.tar.gz
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
+      - run: |
+          ./Tools/pcre-build.sh
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
+      - run: |
+          sh autogen.sh
+          ./configure \
+            --prefix=${{ github.workspace }}/build-static/ \
+            --program-prefix=ds-
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
+      - run: |
+          make -j
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
+      - run: |
+          make install
+        if: steps.swig-build-cache.outputs.cache-hit != 'true'
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}
+          path: ${{ github.workspace }}/build-static/
+  build-ctc-decoder-Linux:
+    name: "Lin|Build CTC decoder Python package for testing"
+    needs: [ swig_Linux ]
+    runs-on: ubuntu-20.04
+    if: ${{ github.event_name == 'pull_request' }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+      - run: |
+          python --version
+          pip --version
+      - uses: actions/download-artifact@v2
+        with:
+          name: "swig_Linux"
+          path: ${{ github.workspace }}/native_client/ds-swig/
+      - name: Link ds-swig into swig
+        run: |
+          ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
+          ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
+          chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
+      - id: get_numpy
+        uses: ./.github/actions/numpy_vers
+        with:
+          pyver: 3.6.8
+      - name: Make decoder package
+        run: |
+          NUMPY_BUILD_VERSION=${{ steps.get_numpy.outputs.build_version }} \
+          NUMPY_DEP_VERSION=${{ steps.get_numpy.outputs.dep_version }} \
+          make -C native_client/ctcdecode/ \
+            NUM_PROCESSES=$(nproc) \
+            bindings
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "ds_ctcdecoder-Linux-test.whl"
+          path: ${{ github.workspace }}/native_client/ctcdecode/dist/*.whl
+      - run: |
+          make -C native_client/ctcdecode clean-keep-third-party
+  train-test-model-Linux:
+    name: "Lin|Train a test model"
+    needs: [ "build-ctc-decoder-Linux" ]
+    runs-on: ubuntu-20.04
+    if: ${{ github.event_name == 'pull_request' }}
+    strategy:
+      matrix:
+        bitrate: ["8k", "16k"]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+      - uses: actions/download-artifact@v2
+        with:
+          name: "ds_ctcdecoder-Linux-test.whl"
+      - run: |
+          python --version
+          pip --version
+      - run: |
+          pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6
+      - run: |
+          pip install ds_ctcdecoder-*-cp36-cp36m-*_x86_64.whl
+          DS_NODECODER=y pip install --upgrade .
+      - run: |
+          bits=""
+          if [ "${{ matrix.bitrate }}" = "8k" ]; then
+            bits=8000
+          fi
+          if [ "${{ matrix.bitrate }}" = "16k"  ]; then
+            bits=16000
+          fi
+
+          # Easier to rename to that we can exercize the LDC93S1 importer code to
+          # generate the CSV file.
+          echo "Moving ${bits} to LDC93S1.wav"
+          mv data/smoke_test/LDC93S1_pcms16le_1_${bits}.wav data/smoke_test/LDC93S1.wav
+
+          ./bin/run-ci-ldc93s1_new.sh 249 ${bits}
+          ./bin/run-ci-ldc93s1_tflite.sh ${bits}
+      - run: |
+          wget -O temp.zip https://github.com/coqui-ai/STT/releases/download/v0.9.3/convert_graphdef_memmapped_format.linux.amd64.zip
+          unzip temp.zip
+          rm temp.zip
+          chmod +x convert_graphdef_memmapped_format
+          ./convert_graphdef_memmapped_format --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm
+      - run: |
+          tar -cf - \
+            -C /tmp/ckpt/ . \
+            | xz -9 -T0 > /tmp/checkpoint.tar.xz
+      - run: |
+          mkdir -p ${{ github.workspace }}/tmp/
+          cp /tmp/train*/output_graph.* /tmp/checkpoint.tar.xz ${{ github.workspace }}/tmp/
+      - run: |
+          ls -hal /tmp/ ${{ github.workspace }}/tmp/
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "test-model.tf-${{ matrix.bitrate }}.zip"
+          path: ${{ github.workspace }}/tmp/output_graph.pb*
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "test-model.tflite-${{ matrix.bitrate }}.zip"
+          path: ${{ github.workspace }}/tmp/output_graph.tflite
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "test-checkpoint.${{ matrix.bitrate }}.zip"
+          path: ${{ github.workspace }}/tmp/checkpoint.tar.xz
+  tensorflow_opt-Linux:
+    name: "Lin|Check TensorFlow cache"
+    runs-on: ubuntu-20.04
+    outputs:
+      status: ${{ steps.check_artifact_exists.outputs.status }}
+      cache_key: ${{ steps.get_cache_key.outputs.key }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - id: get_cache_key
+        uses: ./.github/actions/get_cache_key
+        with:
+          extras: "2"
+      - id: check_artifact_exists
+        uses: ./.github/actions/check_artifact_exists
+        with:
+          name: ${{ steps.get_cache_key.outputs.key }}
+  build-tensorflow-Linux:
+    name: "Lin|Build TensorFlow (opt)"
+    needs: tensorflow_opt-Linux
+    runs-on: ubuntu-20.04
+    steps:
+      - run: true
+        if: needs.tensorflow_opt-Linux.outputs.status == 'found'
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          submodules: 'recursive'
+        if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
+      - run: |
+          sudo apt-get install -y --no-install-recommends pixz
+        if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
+      - uses: ./.github/actions/setup-tensorflow
+        if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
+      - uses: ./.github/actions/build-tensorflow
+        with:
+          flavor: "--linux-cpu"
+        if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
+      - uses: ./.github/actions/package-tensorflow
+        if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}
+          path: ${{ github.workspace }}/artifacts/home.tar.xz
+        if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
+  build-lib_Linux:
+    name: "Lin|Build libstt+client"
+    runs-on: ubuntu-20.04
+    needs: [ build-tensorflow-Linux, tensorflow_opt-Linux ]
+    strategy:
+      matrix:
+        build-flavor: ["tf", "tflite"]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: actions/download-artifact@v2
+        with:
+          name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}
+          path: ${{ github.workspace }}/
+        if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
+      - uses: ./.github/actions/check_artifact_exists
+        with:
+          name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}
+          path: ${{ github.workspace }}/
+          download: true
+        if: needs.tensorflow_opt-Linux.outputs.status == 'found'
+      - run: |
+          tar --skip-old-files -xf ${{ github.workspace }}/home.tar.xz
+          rm ${{ github.workspace }}/home.tar.xz
+      - run: |
+          sudo apt-get install -y --no-install-recommends make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev
+      - run: |
+          git status
+      - uses: ./.github/actions/host-build
+        with:
+          flavor: ${{ matrix.build-flavor }}
+      - uses: ./.github/actions/package
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "native_client.${{ matrix.build-flavor }}.Linux.tar.xz"
+          path: ${{ github.workspace }}/artifacts/native_client.tar.xz
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "libstt.${{ matrix.build-flavor }}.zip"
+          path: ${{ github.workspace }}/artifacts/libstt.zip
+  build-python-Linux:
+    name: "Lin|Build Python bindings"
+    runs-on: ubuntu-20.04
+    needs: [ build-lib_Linux, swig_Linux ]
+    strategy:
+      matrix:
+        build-flavor: ["tf", "tflite"]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/download-artifact@v2
+        with:
+          name: "native_client.${{ matrix.build-flavor }}.Linux.tar.xz"
+          path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
+      - run: |
+          cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
+          tar xf native_client.tar.xz
+          ls -hal
+          cd ${{ github.workspace }}/
+      - uses: actions/download-artifact@v2
+        with:
+          name: "swig_Linux"
+          path: ${{ github.workspace }}/native_client/ds-swig/
+      - name: Link ds-swig into swig
+        run: |
+          ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
+          ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
+          chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - id: get_numpy
+        uses: ./.github/actions/numpy_vers
+        with:
+          pyver: ${{ matrix.python-version }}
+      - uses: ./.github/actions/python-build
+        with:
+          build_flavor: ${{ matrix.build-flavor }}
+          numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
+          numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}-Linux.whl"
+          path: ${{ github.workspace }}/wheels/*.whl
+  build-nodejs-Linux:
+    name: "Lin|Build NodeJS and ElectronJS"
+    runs-on: ubuntu-20.04
+    needs: [ build-lib_Linux, swig_Linux ]
+    strategy:
+      matrix:
+        build-flavor: ["tf", "tflite"]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/download-artifact@v2
+        with:
+          name: "native_client.${{ matrix.build-flavor }}.Linux.tar.xz"
+          path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
+      - run: |
+          cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
+          tar xf native_client.tar.xz
+          ls -hal
+          cd ${{ github.workspace }}/
+      - uses: actions/download-artifact@v2
+        with:
+          name: "swig_Linux"
+          path: ${{ github.workspace }}/native_client/ds-swig/
+      - name: Link ds-swig into swig
+        run: |
+          ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
+          ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
+          chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 12
+      - uses: actions/cache@v2
+        id: node-headers-cache
+        with:
+          path: native_client/javascript/headers/nodejs/
+          key: node-headers-10.0.0_15.0.0
+      - uses: actions/cache@v2
+        id: electron-headers-cache
+        with:
+          path: native_client/javascript/headers/electronjs/
+          key: electron-headers-5.0.13_12.0.0
+      - uses: ./.github/actions/node-build
+        with:
+          nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0"
+          electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "nodewrapper-${{ matrix.build-flavor }}-Linux_amd64.tar.gz"
+          path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz
+      - uses: actions/upload-artifact@v2
+        with:
+          name: "stt_intermediate-${{ matrix.build-flavor }}-Linux.tgz"
+          path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz
+  test-cpp-Linux:
+    name: "Lin|Test C++ binary"
+    runs-on: ubuntu-20.04
+    needs: [ build-lib_Linux, train-test-model-Linux ]
+    if: ${{ github.event_name == 'pull_request' }}
+    strategy:
+      matrix:
+        build-flavor: ["tf", "tflite"]
+        models: ["test", "prod"]
+        bitrate: ["8k", "16k"]
+    env:
+      CI_TMP_DIR: ${{ github.workspace }}/tmp/
+      STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
+      STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/download-artifact@v2
+        with:
+          name: "native_client.${{ matrix.build-flavor }}.Linux.tar.xz"
+          path: ${{ env.CI_TMP_DIR }}
+      - run: |
+          cd ${{ env.CI_TMP_DIR }}
+          mkdir ds && cd ds && tar xf ../native_client.tar.xz
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.CI_TMP_DIR }}
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "cpp"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+  test-py-Linux:
+    name: "Lin|Test Python bindings"
+    runs-on: ubuntu-20.04
+    needs: [ build-python-Linux, train-test-model-Linux ]
+    if: ${{ github.event_name == 'pull_request' }}
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9]
+        build-flavor: ["tf", "tflite"]
+        models: ["test", "prod"]
+        bitrate: ["8k", "16k"]
+    env:
+      CI_TMP_DIR: ${{ github.workspace }}/tmp/
+      STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
+      STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: |
+          sudo apt-get install -y --no-install-recommends sox
+      - uses: actions/download-artifact@v2
+        with:
+          name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}-Linux.whl"
+          path: ${{ env.CI_TMP_DIR }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.CI_TMP_DIR }}
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+          pip3 install --only-binary :all: --upgrade ${{ env.CI_TMP_DIR }}/stt*.whl
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "python"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+  test-nodejs-Linux:
+    name: "Lin|Test NodeJS bindings"
+    runs-on: ubuntu-20.04
+    needs: [ build-nodejs-Linux, train-test-model-Linux ]
+    if: ${{ github.event_name == 'pull_request' }}
+    strategy:
+      matrix:
+        # https://nodejs.org/en/about/releases/
+        nodejs-version: [10, 12, 14, 15]
+        build-flavor: ["tf", "tflite"]
+        models: ["test"]
+        bitrate: ["16k"]
+    env:
+      CI_TMP_DIR: ${{ github.workspace }}/tmp/
+      STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
+      STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.nodejs-version }}
+      - run: |
+          sudo apt-get install -y --no-install-recommends sox
+      - uses: actions/download-artifact@v2
+        with:
+          name: "stt_intermediate-${{ matrix.build-flavor }}-Linux.tgz"
+          path: ${{ env.CI_TMP_DIR }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.CI_TMP_DIR }}
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - uses: actions/cache@v2
+        id: node-modules-cache
+        with:
+          path: ~/.npm/
+          key: node-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }}
+      - name: Install STT package
+        run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+          npm install --verbose ${{ env.CI_TMP_DIR }}/stt*.tgz
+      - run: |
+          ls -hal node_modules/stt* node_modules/.bin/
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "node"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+  test-electronjs-Linux:
+    name: "Lin|Test ElectronJS bindings"
+    runs-on: ubuntu-20.04
+    needs: [ build-nodejs-Linux, train-test-model-Linux ]
+    if: ${{ github.event_name == 'pull_request' }}
+    strategy:
+      matrix:
+        electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0]
+        build-flavor: ["tf", "tflite"]
+        models: ["test"]
+        bitrate: ["16k"]
+    env:
+      CI_TMP_DIR: ${{ github.workspace }}/tmp/
+      STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
+      STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 12
+      - run: |
+          sudo apt-get install -y --no-install-recommends sox
+      - uses: actions/download-artifact@v2
+        with:
+          name: "stt_intermediate-${{ matrix.build-flavor }}-Linux.tgz"
+          path: ${{ env.CI_TMP_DIR }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.CI_TMP_DIR }}
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - uses: actions/cache@v2
+        id: electron-modules-cache
+        with:
+          path: ~/.npm/
+          key: electron-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }}
+      - name: Install STT package
+        run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+          npm install ${{ env.CI_TMP_DIR }}/stt*.tgz
+      - run: |
+          npm install electron@${{ matrix.electronjs-version }}
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "electronjs"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+        timeout-minutes: 5
   # macOS jobs
   swig_macOS:
     name: "Mac|Build SWIG"
@@ -132,7 +685,7 @@ jobs:
             bindings
       - uses: actions/upload-artifact@v2
         with:
-          name: "ds_ctcdecoder-test.whl"
+          name: "ds_ctcdecoder-macOS-test.whl"
           path: ${{ github.workspace }}/native_client/ctcdecode/dist/*.whl
       - run: |
           make -C native_client/ctcdecode clean-keep-third-party
@@ -153,7 +706,7 @@ jobs:
           python-version: 3.6
       - uses: actions/download-artifact@v2
         with:
-          name: "ds_ctcdecoder-test.whl"
+          name: "ds_ctcdecoder-macOS-test.whl"
       - run: |
           python --version
           pip --version
@@ -593,47 +1146,8 @@ jobs:
           model-kind: ${{ matrix.models }}
         timeout-minutes: 5
   # Windows jobs
-  swig_Windows:
-    name: "Win|Build SWIG"
-    runs-on: windows-2019
-    steps:
-      - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
-        run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
-      - uses: msys2/setup-msys2@v2
-        with:
-          msystem: MSYS
-          update: true
-          install: >-
-            autoconf
-            automake
-            bison
-            gcc
-            git
-            make
-            pcre-devel
-      - uses: actions/checkout@v2
-        with:
-          repository: "lissyx/swig"
-          ref: "fec7d5d3179833e37759ffc6532f86344982e26a"
-      - run: |
-          mkdir -p build-static/
-      - run: |
-          export CFLAGS="-static-libgcc -static-libstdc++"
-          export CXXFLAGS="-static-libgcc -static-libstdc++"
-          sh autogen.sh
-          ./configure \
-            --prefix=`pwd`/build-static/ \
-            --program-prefix=ds-
-      - run: |
-          make -j
-      - run: |
-          make install
-      - uses: actions/upload-artifact@v2
-        with:
-          name: ${{ github.job }}
-          path: ${{ github.workspace }}/build-static/
   build-ctc-decoder-windows:
-    name: "Win|Build CTC decoder Python package for testing"
+    name: "Win|Build CTC decoder Python package"
     needs: [swig_Windows_crosscompiled]
     runs-on: windows-2019
     steps:
@@ -926,7 +1440,7 @@ jobs:
   test-cpp-Windows:
     name: "Win|Test C++ binary"
     runs-on: windows-2019
-    needs: [build-lib_Windows, train-test-model-macOS]
+    needs: [build-lib_Windows, train-test-model-Linux]
     strategy:
       matrix:
         build-flavor: ["tf", "tflite"]
@@ -974,7 +1488,7 @@ jobs:
   test-py-Windows:
     name: "Win|Test Python bindings"
     runs-on: windows-2019
-    needs: [ build-python-Windows, train-test-model-macOS ]
+    needs: [ build-python-Windows, train-test-model-Linux ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
       matrix:
@@ -1031,7 +1545,7 @@ jobs:
   test-nodejs-Windows:
     name: "Win|Test NodeJS bindings"
     runs-on: windows-2019
-    needs: [ build-nodejs-Windows, train-test-model-macOS ]
+    needs: [ build-nodejs-Windows, train-test-model-Linux ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
       matrix:
@@ -1096,7 +1610,7 @@ jobs:
   test-electronjs-Windows:
     name: "Win|Test ElectronJS bindings"
     runs-on: windows-2019
-    needs: [ build-nodejs-Windows, train-test-model-macOS ]
+    needs: [ build-nodejs-Windows, train-test-model-Linux ]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
       matrix:
@@ -1165,7 +1679,7 @@ jobs:
   repackage-nodejs-allplatforms:
     name: "Repackage NodeJS / ElectronJS for multiplatforms"
     runs-on: ubuntu-20.04
-    needs: [build-nodejs-macOS, build-nodejs-Windows]
+    needs: [build-nodejs-macOS, build-nodejs-Windows, build-nodejs-Linux]
     strategy:
       matrix:
         build-flavor: ["tf", "tflite"]
@@ -1184,10 +1698,15 @@ jobs:
         with:
           name: "nodewrapper-${{ matrix.build-flavor }}-Windows_amd64.tar.gz"
           path: /tmp/nodewrapper-Windows_amd64/
+      - uses: actions/download-artifact@v2
+        with:
+          name: "nodewrapper-${{ matrix.build-flavor }}-Linux_amd64.tar.gz"
+          path: /tmp/nodewrapper-Linux_amd64/
       - name: Extract nodewrapper archives
         run: |
           tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-macOS_amd64/wrapper.tar.gz
           tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-Windows_amd64/wrapper.tar.gz
+          tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-Linux_amd64/wrapper.tar.gz
       - run: |
           PROJECT_NAME="stt"
           if [ "${{ matrix.build-flavor }}" = "tflite" ]; then
@@ -1198,6 +1717,118 @@ jobs:
         with:
           name: "stt-${{ matrix.build-flavor }}.tgz"
           path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz
+  test-nodejs_all-Linux:
+    name: "Lin|Test MultiArchPlatform NodeJS bindings"
+    runs-on: ubuntu-20.04
+    needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
+    if: ${{ github.event_name == 'pull_request' }}
+    strategy:
+      matrix:
+        # https://nodejs.org/en/about/releases/
+        nodejs-version: [10, 15]
+        build-flavor: ["tf", "tflite"]
+        models: ["test", "prod"]
+        bitrate: ["8k", "16k"]
+    env:
+      CI_TMP_DIR: ${{ github.workspace }}/tmp/
+      STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
+      STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.nodejs-version }}
+      - run: |
+          sudo apt-get install -y --no-install-recommends sox
+      - uses: actions/download-artifact@v2
+        with:
+          name: "stt-${{ matrix.build-flavor }}.tgz"
+          path: ${{ env.CI_TMP_DIR }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.CI_TMP_DIR }}
+        if: matrix.models == 'test'
+      - uses: actions/cache@v2
+        id: node-modules-cache
+        with:
+          path: ~/.npm/
+          key: node-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }}
+      - run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - name: Install STT package
+        run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+          npm install --verbose ${{ env.CI_TMP_DIR }}/stt*.tgz
+      - run: |
+          ls -hal node_modules/stt* node_modules/.bin/
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "node"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+  test-electronjs_all-Linux:
+    name: "Lin|Test MultiArchPlatform ElectronJS bindings"
+    runs-on: ubuntu-20.04
+    needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
+    if: ${{ github.event_name == 'pull_request' }}
+    strategy:
+      matrix:
+        electronjs-version: [5.0.13, 12.0.0]
+        build-flavor: ["tf", "tflite"]
+        models: ["test", "prod"]
+        bitrate: ["8k", "16k"]
+    env:
+      CI_TMP_DIR: ${{ github.workspace }}/tmp/
+      STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
+      STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
+      STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
+      EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 12
+      - run: |
+          sudo apt-get install -y --no-install-recommends sox
+      - uses: actions/download-artifact@v2
+        with:
+          name: "stt-${{ matrix.build-flavor }}.tgz"
+          path: ${{ env.CI_TMP_DIR }}
+      - uses: actions/download-artifact@v2
+        with:
+          name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip"
+          path: ${{ env.CI_TMP_DIR }}
+        if: matrix.models == 'test'
+      - run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+        if: matrix.models == 'test'
+      - uses: actions/cache@v2
+        id: electron-modules-cache
+        with:
+          path: ~/.npm/
+          key: electron-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }}
+      - name: Install STT package
+        run: |
+          ls -hal ${{ env.CI_TMP_DIR }}/
+          npm install ${{ env.CI_TMP_DIR }}/stt*.tgz
+      - run: |
+          npm install electron@${{ matrix.electronjs-version }}
+      - uses: ./.github/actions/run-tests
+        with:
+          runtime: "electronjs"
+          build-flavor: ${{ matrix.build-flavor }}
+          bitrate: ${{ matrix.bitrate }}
+          model-kind: ${{ matrix.models }}
+        timeout-minutes: 5
   test-nodejs_all-macOS:
     name: "Mac|Test MultiArchPlatform NodeJS bindings"
     runs-on: macos-10.15
@@ -1309,7 +1940,7 @@ jobs:
   test-nodejs_all-Windows:
     name: "Win|Test MultiArchPlatform NodeJS bindings"
     runs-on: windows-2019
-    needs: [repackage-nodejs-allplatforms, train-test-model-macOS]
+    needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
       matrix:
@@ -1375,7 +2006,7 @@ jobs:
   test-electronjs_all-Windows:
     name: "Win|Test MultiArchPlatform ElectronJS bindings"
     runs-on: windows-2019
-    needs: [repackage-nodejs-allplatforms, train-test-model-macOS]
+    needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
     if: ${{ github.event_name == 'pull_request' }}
     strategy:
       matrix:

ci_scripts/all-vars.sh

@@ -4,7 +4,7 @@ set -xe
 
 export OS=$(uname)
 if [ "${OS}" = "Linux" ]; then
-    export DS_ROOT_TASK=${HOME}
+    export DS_ROOT_TASK=${CI_TASK_DIR}
     export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root"
     export DS_CPU_COUNT=$(nproc)
 fi;

ci_scripts/electronjs-tests-prod.sh

@@ -28,6 +28,12 @@ export_node_bin_path
 which electron
 which node
 
+if [ "${OS}" = "Linux" ]; then
+  export DISPLAY=':99.0'
+  sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+  xvfb_process=$!
+fi
+
 node --version
 
 stt --version
@@ -35,3 +41,8 @@ stt --version
 check_runtime_electronjs
 
 run_electronjs_prod_inference_tests "${bitrate}"
+
+if [ "${OS}" = "Linux" ]; then
+  sleep 1
+  sudo kill -9 ${xvfb_process} || true
+fi

ci_scripts/electronjs-tests.sh

@@ -21,6 +21,12 @@ export_node_bin_path
 which electron
 which node
 
+if [ "${OS}" = "Linux" ]; then
+  export DISPLAY=':99.0'
+  sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+  xvfb_process=$!
+fi
+
 node --version
 
 stt --version
@@ -28,3 +34,8 @@ stt --version
 check_runtime_electronjs
 
 run_electronjs_inference_tests
+
+if [ "${OS}" = "Linux" ]; then
+  sleep 1
+  sudo kill -9 ${xvfb_process} || true
+fi

ci_scripts/electronjs_tflite-tests-prod.sh

@@ -28,6 +28,12 @@ export_node_bin_path
 which electron
 which node
 
+if [ "${OS}" = "Linux" ]; then
+  export DISPLAY=':99.0'
+  sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+  xvfb_process=$!
+fi
+
 node --version
 
 stt --version
@@ -35,3 +41,8 @@ stt --version
 check_runtime_electronjs
 
 run_electronjs_prodtflite_inference_tests "${bitrate}"
+
+if [ "${OS}" = "Linux" ]; then
+  sleep 1
+  sudo kill -9 ${xvfb_process} || true
+fi

ci_scripts/electronjs_tflite-tests.sh

@@ -25,6 +25,12 @@ export_node_bin_path
 which electron
 which node
 
+if [ "${OS}" = "Linux" ]; then
+  export DISPLAY=':99.0'
+  sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+  xvfb_process=$!
+fi
+
 node --version
 
 stt --version
@@ -32,3 +38,8 @@ stt --version
 check_runtime_electronjs
 
 run_electronjs_inference_tests
+
+if [ "${OS}" = "Linux" ]; then
+  sleep 1
+  sudo kill -9 ${xvfb_process} || true
+fi

ci_scripts/tf-vars.sh

@@ -4,7 +4,7 @@ set -ex
 
 export OS=$(uname)
 if [ "${OS}" = "Linux" ]; then
-    export DS_ROOT_TASK=$(/usr/bin/realpath "${HOME}")
+    export DS_ROOT_TASK=${CI_TASK_DIR}
 
     BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-linux-x86_64.sh
     BAZEL_SHA256=7ba815cbac712d061fe728fef958651512ff394b2708e89f79586ec93d1185ed