diff --git a/.github/actions/check_artifact_exists/dist/index.js b/.github/actions/check_artifact_exists/dist/index.js index c33d562f..b20ce52b 100644 Binary files a/.github/actions/check_artifact_exists/dist/index.js and b/.github/actions/check_artifact_exists/dist/index.js differ diff --git a/.github/actions/check_artifact_exists/main.js b/.github/actions/check_artifact_exists/main.js index e7608aa1..b86b5e3e 100644 --- a/.github/actions/check_artifact_exists/main.js +++ b/.github/actions/check_artifact_exists/main.js @@ -8,49 +8,28 @@ const { throttling } = require('@octokit/plugin-throttling'); const { GitHub } = require('@actions/github/lib/utils'); async function getGoodArtifacts(client, owner, repo, name) { - const goodWorkflowArtifacts = await client.paginate( - "GET /repos/{owner}/{repo}/actions/runs/{run_id}/artifacts", - { - owner: owner, - repo: repo, - run_id: github.context.runId, - }, - (workflowArtifacts) => { - // console.log(" ==> workflowArtifacts", workflowArtifacts); - return workflowArtifacts.data.filter((a) => { - // console.log("==> Artifact check", a); - return a.name == name - }) - } - ); - - console.log("==> maybe goodWorkflowArtifacts:", goodWorkflowArtifacts); - if (goodWorkflowArtifacts.length > 0) { - return goodWorkflowArtifacts; - } - const goodRepoArtifacts = await client.paginate( "GET /repos/{owner}/{repo}/actions/artifacts", { owner: owner, repo: repo, + per_page: 100, }, - (repoArtifacts) => { + (repoArtifacts, done) => { // console.log(" ==> repoArtifacts", repoArtifacts); - return repoArtifacts.data.filter((a) => { + const goodArtifacts = repoArtifacts.data.filter((a) => { // console.log("==> Artifact check", a); return a.name == name - }) + }); + if (goodArtifacts.length > 0) { + done(); + } + return goodArtifacts; } ); console.log("==> maybe goodRepoArtifacts:", goodRepoArtifacts); - if (goodRepoArtifacts.length > 0) { - return goodRepoArtifacts; - } - - // We have not been able to find a repo artifact, it's really no good news - return []; + return goodRepoArtifacts; } async function main() { diff --git a/.github/actions/check_artifact_exists/package-lock.json b/.github/actions/check_artifact_exists/package-lock.json index 3a70ac66..fd188e3c 100644 --- a/.github/actions/check_artifact_exists/package-lock.json +++ b/.github/actions/check_artifact_exists/package-lock.json @@ -79,7 +79,8 @@ "@octokit/openapi-types": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-6.0.0.tgz", - "integrity": "sha512-CnDdK7ivHkBtJYzWzZm7gEkanA7gKH6a09Eguz7flHw//GacPJLmkHA3f3N++MJmlxD1Fl+mB7B32EEpSCwztQ==" + "integrity": "sha512-CnDdK7ivHkBtJYzWzZm7gEkanA7gKH6a09Eguz7flHw//GacPJLmkHA3f3N++MJmlxD1Fl+mB7B32EEpSCwztQ==", + "dev": true }, "@octokit/plugin-paginate-rest": { "version": "2.13.3", @@ -104,6 +105,7 @@ "version": "3.4.1", "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-3.4.1.tgz", "integrity": "sha512-qCQ+Z4AnL9OrXvV59EH3GzPxsB+WyqufoCjiCJXJxTbnt3W+leXbXw5vHrMp4NG9ltw00McFWIxIxNQAzLNoTA==", + "dev": true, "requires": { "@octokit/types": "^6.0.1", "bottleneck": "^2.15.3" @@ -138,6 +140,7 @@ "version": "6.13.0", "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.13.0.tgz", "integrity": "sha512-W2J9qlVIU11jMwKHUp5/rbVUeErqelCsO5vW5PKNb7wAXQVUz87Rc+imjlEvpvbH8yUb+KHmv8NEjVZdsdpyxA==", + "dev": true, "requires": { "@octokit/openapi-types": "^6.0.0" } @@ -163,7 +166,8 @@ "bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", - "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==", + "dev": true }, "deprecation": { "version": "2.3.1", diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 9044177d..5c684dc8 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -25,33 +25,586 @@ jobs: name: "Lin|Build SWIG for Windows" runs-on: ubuntu-20.04 env: - CFLAGS: "-static-libgcc -static-libstdc++" - CXXFLAGS: "-static-libgcc -static-libstdc++" + swig_hash: "90cdbee6a69d13b39d734083b9f91069533b0d7b" steps: - - run: | - sudo apt-get install -y --no-install-recommends autoconf automake bison build-essential mingw-w64 - uses: actions/checkout@v2 with: - repository: "lissyx/swig" - ref: "fec7d5d3179833e37759ffc6532f86344982e26a" + repository: "swig/swig" + ref: ${{ env.swig_hash }} - run: | mkdir -p build-static/ + - uses: actions/cache@v2 + id: swig-build-cache + with: + path: build-static/ + key: swig-win-3-${{ env.swig_hash }} + - run: | + sudo apt-get install -y --no-install-recommends autoconf automake bison build-essential mingw-w64 + if: steps.swig-build-cache.outputs.cache-hit != 'true' - run: | curl -sSL https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz > pcre-8.43.tar.gz + if: steps.swig-build-cache.outputs.cache-hit != 'true' + - run: | ./Tools/pcre-build.sh --host=x86_64-w64-mingw32 + if: steps.swig-build-cache.outputs.cache-hit != 'true' - run: | sh autogen.sh - ./configure --host=x86_64-w64-mingw32 \ + CFLAGS="-static-libgcc -static-libstdc++" \ + CXXFLAGS="-static-libgcc -static-libstdc++" \ + ./configure \ + --host=x86_64-w64-mingw32 \ --prefix=`pwd`/build-static/ \ --program-prefix=ds- + if: steps.swig-build-cache.outputs.cache-hit != 'true' - run: | make -j + if: steps.swig-build-cache.outputs.cache-hit != 'true' - run: | make install + if: steps.swig-build-cache.outputs.cache-hit != 'true' - uses: actions/upload-artifact@v2 with: name: ${{ github.job }} path: ${{ github.workspace }}/build-static/ + swig_Linux: + name: "Lin|Build SWIG" + runs-on: ubuntu-20.04 + env: + swig_hash: "90cdbee6a69d13b39d734083b9f91069533b0d7b" + steps: + - uses: actions/checkout@v2 + with: + repository: "swig/swig" + ref: ${{ env.swig_hash }} + - run: | + mkdir -p build-static/ + - uses: actions/cache@v2 + id: swig-build-cache + with: + path: build-static/ + key: swig-2-${{ runner.os }}-${{ env.swig_hash }} + - run: | + sudo apt-get install -y --no-install-recommends autoconf automake bison build-essential + if: steps.swig-build-cache.outputs.cache-hit != 'true' + - run: | + curl -sSL https://ftp.pcre.org/pub/pcre/pcre-8.43.tar.gz > pcre-8.43.tar.gz + if: steps.swig-build-cache.outputs.cache-hit != 'true' + - run: | + ./Tools/pcre-build.sh + if: steps.swig-build-cache.outputs.cache-hit != 'true' + - run: | + sh autogen.sh + ./configure \ + --prefix=${{ github.workspace }}/build-static/ \ + --program-prefix=ds- + if: steps.swig-build-cache.outputs.cache-hit != 'true' + - run: | + make -j + if: steps.swig-build-cache.outputs.cache-hit != 'true' + - run: | + make install + if: steps.swig-build-cache.outputs.cache-hit != 'true' + - uses: actions/upload-artifact@v2 + with: + name: ${{ github.job }} + path: ${{ github.workspace }}/build-static/ + build-ctc-decoder-Linux: + name: "Lin|Build CTC decoder Python package for testing" + needs: [ swig_Linux ] + runs-on: ubuntu-20.04 + if: ${{ github.event_name == 'pull_request' }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + - run: | + python --version + pip --version + - uses: actions/download-artifact@v2 + with: + name: "swig_Linux" + path: ${{ github.workspace }}/native_client/ds-swig/ + - name: Link ds-swig into swig + run: | + ls -hal ${{ github.workspace }}/native_client/ds-swig/bin + ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig + chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig + - id: get_numpy + uses: ./.github/actions/numpy_vers + with: + pyver: 3.6.8 + - name: Make decoder package + run: | + NUMPY_BUILD_VERSION=${{ steps.get_numpy.outputs.build_version }} \ + NUMPY_DEP_VERSION=${{ steps.get_numpy.outputs.dep_version }} \ + make -C native_client/ctcdecode/ \ + NUM_PROCESSES=$(nproc) \ + bindings + - uses: actions/upload-artifact@v2 + with: + name: "ds_ctcdecoder-Linux-test.whl" + path: ${{ github.workspace }}/native_client/ctcdecode/dist/*.whl + - run: | + make -C native_client/ctcdecode clean-keep-third-party + train-test-model-Linux: + name: "Lin|Train a test model" + needs: [ "build-ctc-decoder-Linux" ] + runs-on: ubuntu-20.04 + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + bitrate: ["8k", "16k"] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + - uses: actions/download-artifact@v2 + with: + name: "ds_ctcdecoder-Linux-test.whl" + - run: | + python --version + pip --version + - run: | + pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 + - run: | + pip install ds_ctcdecoder-*-cp36-cp36m-*_x86_64.whl + DS_NODECODER=y pip install --upgrade . + - run: | + bits="" + if [ "${{ matrix.bitrate }}" = "8k" ]; then + bits=8000 + fi + if [ "${{ matrix.bitrate }}" = "16k" ]; then + bits=16000 + fi + + # Easier to rename to that we can exercize the LDC93S1 importer code to + # generate the CSV file. + echo "Moving ${bits} to LDC93S1.wav" + mv data/smoke_test/LDC93S1_pcms16le_1_${bits}.wav data/smoke_test/LDC93S1.wav + + ./bin/run-ci-ldc93s1_new.sh 249 ${bits} + ./bin/run-ci-ldc93s1_tflite.sh ${bits} + - run: | + wget -O temp.zip https://github.com/coqui-ai/STT/releases/download/v0.9.3/convert_graphdef_memmapped_format.linux.amd64.zip + unzip temp.zip + rm temp.zip + chmod +x convert_graphdef_memmapped_format + ./convert_graphdef_memmapped_format --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm + - run: | + tar -cf - \ + -C /tmp/ckpt/ . \ + | xz -9 -T0 > /tmp/checkpoint.tar.xz + - run: | + mkdir -p ${{ github.workspace }}/tmp/ + cp /tmp/train*/output_graph.* /tmp/checkpoint.tar.xz ${{ github.workspace }}/tmp/ + - run: | + ls -hal /tmp/ ${{ github.workspace }}/tmp/ + - uses: actions/upload-artifact@v2 + with: + name: "test-model.tf-${{ matrix.bitrate }}.zip" + path: ${{ github.workspace }}/tmp/output_graph.pb* + - uses: actions/upload-artifact@v2 + with: + name: "test-model.tflite-${{ matrix.bitrate }}.zip" + path: ${{ github.workspace }}/tmp/output_graph.tflite + - uses: actions/upload-artifact@v2 + with: + name: "test-checkpoint.${{ matrix.bitrate }}.zip" + path: ${{ github.workspace }}/tmp/checkpoint.tar.xz + tensorflow_opt-Linux: + name: "Lin|Check TensorFlow cache" + runs-on: ubuntu-20.04 + outputs: + status: ${{ steps.check_artifact_exists.outputs.status }} + cache_key: ${{ steps.get_cache_key.outputs.key }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - id: get_cache_key + uses: ./.github/actions/get_cache_key + with: + extras: "2" + - id: check_artifact_exists + uses: ./.github/actions/check_artifact_exists + with: + name: ${{ steps.get_cache_key.outputs.key }} + build-tensorflow-Linux: + name: "Lin|Build TensorFlow (opt)" + needs: tensorflow_opt-Linux + runs-on: ubuntu-20.04 + steps: + - run: true + if: needs.tensorflow_opt-Linux.outputs.status == 'found' + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: 'recursive' + if: needs.tensorflow_opt-Linux.outputs.status == 'missing' + - run: | + sudo apt-get install -y --no-install-recommends pixz + if: needs.tensorflow_opt-Linux.outputs.status == 'missing' + - uses: ./.github/actions/setup-tensorflow + if: needs.tensorflow_opt-Linux.outputs.status == 'missing' + - uses: ./.github/actions/build-tensorflow + with: + flavor: "--linux-cpu" + if: needs.tensorflow_opt-Linux.outputs.status == 'missing' + - uses: ./.github/actions/package-tensorflow + if: needs.tensorflow_opt-Linux.outputs.status == 'missing' + - uses: actions/upload-artifact@v2 + with: + name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }} + path: ${{ github.workspace }}/artifacts/home.tar.xz + if: needs.tensorflow_opt-Linux.outputs.status == 'missing' + build-lib_Linux: + name: "Lin|Build libstt+client" + runs-on: ubuntu-20.04 + needs: [ build-tensorflow-Linux, tensorflow_opt-Linux ] + strategy: + matrix: + build-flavor: ["tf", "tflite"] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v2 + with: + name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }} + path: ${{ github.workspace }}/ + if: needs.tensorflow_opt-Linux.outputs.status == 'missing' + - uses: ./.github/actions/check_artifact_exists + with: + name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }} + path: ${{ github.workspace }}/ + download: true + if: needs.tensorflow_opt-Linux.outputs.status == 'found' + - run: | + tar --skip-old-files -xf ${{ github.workspace }}/home.tar.xz + rm ${{ github.workspace }}/home.tar.xz + - run: | + sudo apt-get install -y --no-install-recommends make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev + - run: | + git status + - uses: ./.github/actions/host-build + with: + flavor: ${{ matrix.build-flavor }} + - uses: ./.github/actions/package + - uses: actions/upload-artifact@v2 + with: + name: "native_client.${{ matrix.build-flavor }}.Linux.tar.xz" + path: ${{ github.workspace }}/artifacts/native_client.tar.xz + - uses: actions/upload-artifact@v2 + with: + name: "libstt.${{ matrix.build-flavor }}.zip" + path: ${{ github.workspace }}/artifacts/libstt.zip + build-python-Linux: + name: "Lin|Build Python bindings" + runs-on: ubuntu-20.04 + needs: [ build-lib_Linux, swig_Linux ] + strategy: + matrix: + build-flavor: ["tf", "tflite"] + python-version: [3.6, 3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/download-artifact@v2 + with: + name: "native_client.${{ matrix.build-flavor }}.Linux.tar.xz" + path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ + - run: | + cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ + tar xf native_client.tar.xz + ls -hal + cd ${{ github.workspace }}/ + - uses: actions/download-artifact@v2 + with: + name: "swig_Linux" + path: ${{ github.workspace }}/native_client/ds-swig/ + - name: Link ds-swig into swig + run: | + ls -hal ${{ github.workspace }}/native_client/ds-swig/bin + ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig + chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - id: get_numpy + uses: ./.github/actions/numpy_vers + with: + pyver: ${{ matrix.python-version }} + - uses: ./.github/actions/python-build + with: + build_flavor: ${{ matrix.build-flavor }} + numpy_build: "${{ steps.get_numpy.outputs.build_version }}" + numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}" + - uses: actions/upload-artifact@v2 + with: + name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}-Linux.whl" + path: ${{ github.workspace }}/wheels/*.whl + build-nodejs-Linux: + name: "Lin|Build NodeJS and ElectronJS" + runs-on: ubuntu-20.04 + needs: [ build-lib_Linux, swig_Linux ] + strategy: + matrix: + build-flavor: ["tf", "tflite"] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/download-artifact@v2 + with: + name: "native_client.${{ matrix.build-flavor }}.Linux.tar.xz" + path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ + - run: | + cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ + tar xf native_client.tar.xz + ls -hal + cd ${{ github.workspace }}/ + - uses: actions/download-artifact@v2 + with: + name: "swig_Linux" + path: ${{ github.workspace }}/native_client/ds-swig/ + - name: Link ds-swig into swig + run: | + ls -hal ${{ github.workspace }}/native_client/ds-swig/bin + ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig + chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig + - uses: actions/setup-node@v2 + with: + node-version: 12 + - uses: actions/cache@v2 + id: node-headers-cache + with: + path: native_client/javascript/headers/nodejs/ + key: node-headers-10.0.0_15.0.0 + - uses: actions/cache@v2 + id: electron-headers-cache + with: + path: native_client/javascript/headers/electronjs/ + key: electron-headers-5.0.13_12.0.0 + - uses: ./.github/actions/node-build + with: + nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0" + electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0" + - uses: actions/upload-artifact@v2 + with: + name: "nodewrapper-${{ matrix.build-flavor }}-Linux_amd64.tar.gz" + path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz + - uses: actions/upload-artifact@v2 + with: + name: "stt_intermediate-${{ matrix.build-flavor }}-Linux.tgz" + path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz + test-cpp-Linux: + name: "Lin|Test C++ binary" + runs-on: ubuntu-20.04 + needs: [ build-lib_Linux, train-test-model-Linux ] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + build-flavor: ["tf", "tflite"] + models: ["test", "prod"] + bitrate: ["8k", "16k"] + env: + CI_TMP_DIR: ${{ github.workspace }}/tmp/ + STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb + STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm + STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/download-artifact@v2 + with: + name: "native_client.${{ matrix.build-flavor }}.Linux.tar.xz" + path: ${{ env.CI_TMP_DIR }} + - run: | + cd ${{ env.CI_TMP_DIR }} + mkdir ds && cd ds && tar xf ../native_client.tar.xz + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.CI_TMP_DIR }} + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + if: matrix.models == 'test' + - uses: ./.github/actions/run-tests + with: + runtime: "cpp" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + test-py-Linux: + name: "Lin|Test Python bindings" + runs-on: ubuntu-20.04 + needs: [ build-python-Linux, train-test-model-Linux ] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + build-flavor: ["tf", "tflite"] + models: ["test", "prod"] + bitrate: ["8k", "16k"] + env: + CI_TMP_DIR: ${{ github.workspace }}/tmp/ + STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb + STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm + STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - run: | + sudo apt-get install -y --no-install-recommends sox + - uses: actions/download-artifact@v2 + with: + name: "stt-${{ matrix.build-flavor }}-${{ matrix.python-version }}-Linux.whl" + path: ${{ env.CI_TMP_DIR }} + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.CI_TMP_DIR }} + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + pip3 install --only-binary :all: --upgrade ${{ env.CI_TMP_DIR }}/stt*.whl + - uses: ./.github/actions/run-tests + with: + runtime: "python" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + test-nodejs-Linux: + name: "Lin|Test NodeJS bindings" + runs-on: ubuntu-20.04 + needs: [ build-nodejs-Linux, train-test-model-Linux ] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + # https://nodejs.org/en/about/releases/ + nodejs-version: [10, 12, 14, 15] + build-flavor: ["tf", "tflite"] + models: ["test"] + bitrate: ["16k"] + env: + CI_TMP_DIR: ${{ github.workspace }}/tmp/ + STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb + STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm + STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.nodejs-version }} + - run: | + sudo apt-get install -y --no-install-recommends sox + - uses: actions/download-artifact@v2 + with: + name: "stt_intermediate-${{ matrix.build-flavor }}-Linux.tgz" + path: ${{ env.CI_TMP_DIR }} + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.CI_TMP_DIR }} + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + if: matrix.models == 'test' + - uses: actions/cache@v2 + id: node-modules-cache + with: + path: ~/.npm/ + key: node-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }} + - name: Install STT package + run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + npm install --verbose ${{ env.CI_TMP_DIR }}/stt*.tgz + - run: | + ls -hal node_modules/stt* node_modules/.bin/ + - uses: ./.github/actions/run-tests + with: + runtime: "node" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + test-electronjs-Linux: + name: "Lin|Test ElectronJS bindings" + runs-on: ubuntu-20.04 + needs: [ build-nodejs-Linux, train-test-model-Linux ] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0] + build-flavor: ["tf", "tflite"] + models: ["test"] + bitrate: ["16k"] + env: + CI_TMP_DIR: ${{ github.workspace }}/tmp/ + STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb + STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm + STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-node@v2 + with: + node-version: 12 + - run: | + sudo apt-get install -y --no-install-recommends sox + - uses: actions/download-artifact@v2 + with: + name: "stt_intermediate-${{ matrix.build-flavor }}-Linux.tgz" + path: ${{ env.CI_TMP_DIR }} + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.CI_TMP_DIR }} + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + if: matrix.models == 'test' + - uses: actions/cache@v2 + id: electron-modules-cache + with: + path: ~/.npm/ + key: electron-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }} + - name: Install STT package + run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + npm install ${{ env.CI_TMP_DIR }}/stt*.tgz + - run: | + npm install electron@${{ matrix.electronjs-version }} + - uses: ./.github/actions/run-tests + with: + runtime: "electronjs" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + timeout-minutes: 5 # macOS jobs swig_macOS: name: "Mac|Build SWIG" @@ -132,7 +685,7 @@ jobs: bindings - uses: actions/upload-artifact@v2 with: - name: "ds_ctcdecoder-test.whl" + name: "ds_ctcdecoder-macOS-test.whl" path: ${{ github.workspace }}/native_client/ctcdecode/dist/*.whl - run: | make -C native_client/ctcdecode clean-keep-third-party @@ -153,7 +706,7 @@ jobs: python-version: 3.6 - uses: actions/download-artifact@v2 with: - name: "ds_ctcdecoder-test.whl" + name: "ds_ctcdecoder-macOS-test.whl" - run: | python --version pip --version @@ -593,47 +1146,8 @@ jobs: model-kind: ${{ matrix.models }} timeout-minutes: 5 # Windows jobs - swig_Windows: - name: "Win|Build SWIG" - runs-on: windows-2019 - steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - - uses: msys2/setup-msys2@v2 - with: - msystem: MSYS - update: true - install: >- - autoconf - automake - bison - gcc - git - make - pcre-devel - - uses: actions/checkout@v2 - with: - repository: "lissyx/swig" - ref: "fec7d5d3179833e37759ffc6532f86344982e26a" - - run: | - mkdir -p build-static/ - - run: | - export CFLAGS="-static-libgcc -static-libstdc++" - export CXXFLAGS="-static-libgcc -static-libstdc++" - sh autogen.sh - ./configure \ - --prefix=`pwd`/build-static/ \ - --program-prefix=ds- - - run: | - make -j - - run: | - make install - - uses: actions/upload-artifact@v2 - with: - name: ${{ github.job }} - path: ${{ github.workspace }}/build-static/ build-ctc-decoder-windows: - name: "Win|Build CTC decoder Python package for testing" + name: "Win|Build CTC decoder Python package" needs: [swig_Windows_crosscompiled] runs-on: windows-2019 steps: @@ -926,7 +1440,7 @@ jobs: test-cpp-Windows: name: "Win|Test C++ binary" runs-on: windows-2019 - needs: [build-lib_Windows, train-test-model-macOS] + needs: [build-lib_Windows, train-test-model-Linux] strategy: matrix: build-flavor: ["tf", "tflite"] @@ -974,7 +1488,7 @@ jobs: test-py-Windows: name: "Win|Test Python bindings" runs-on: windows-2019 - needs: [ build-python-Windows, train-test-model-macOS ] + needs: [ build-python-Windows, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: matrix: @@ -1031,7 +1545,7 @@ jobs: test-nodejs-Windows: name: "Win|Test NodeJS bindings" runs-on: windows-2019 - needs: [ build-nodejs-Windows, train-test-model-macOS ] + needs: [ build-nodejs-Windows, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: matrix: @@ -1096,7 +1610,7 @@ jobs: test-electronjs-Windows: name: "Win|Test ElectronJS bindings" runs-on: windows-2019 - needs: [ build-nodejs-Windows, train-test-model-macOS ] + needs: [ build-nodejs-Windows, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: matrix: @@ -1165,7 +1679,7 @@ jobs: repackage-nodejs-allplatforms: name: "Repackage NodeJS / ElectronJS for multiplatforms" runs-on: ubuntu-20.04 - needs: [build-nodejs-macOS, build-nodejs-Windows] + needs: [build-nodejs-macOS, build-nodejs-Windows, build-nodejs-Linux] strategy: matrix: build-flavor: ["tf", "tflite"] @@ -1184,10 +1698,15 @@ jobs: with: name: "nodewrapper-${{ matrix.build-flavor }}-Windows_amd64.tar.gz" path: /tmp/nodewrapper-Windows_amd64/ + - uses: actions/download-artifact@v2 + with: + name: "nodewrapper-${{ matrix.build-flavor }}-Linux_amd64.tar.gz" + path: /tmp/nodewrapper-Linux_amd64/ - name: Extract nodewrapper archives run: | tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-macOS_amd64/wrapper.tar.gz tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-Windows_amd64/wrapper.tar.gz + tar -C ${{ github.workspace }}/native_client/javascript -xzvf /tmp/nodewrapper-Linux_amd64/wrapper.tar.gz - run: | PROJECT_NAME="stt" if [ "${{ matrix.build-flavor }}" = "tflite" ]; then @@ -1198,6 +1717,118 @@ jobs: with: name: "stt-${{ matrix.build-flavor }}.tgz" path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz + test-nodejs_all-Linux: + name: "Lin|Test MultiArchPlatform NodeJS bindings" + runs-on: ubuntu-20.04 + needs: [repackage-nodejs-allplatforms, train-test-model-Linux] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + # https://nodejs.org/en/about/releases/ + nodejs-version: [10, 15] + build-flavor: ["tf", "tflite"] + models: ["test", "prod"] + bitrate: ["8k", "16k"] + env: + CI_TMP_DIR: ${{ github.workspace }}/tmp/ + STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb + STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm + STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.nodejs-version }} + - run: | + sudo apt-get install -y --no-install-recommends sox + - uses: actions/download-artifact@v2 + with: + name: "stt-${{ matrix.build-flavor }}.tgz" + path: ${{ env.CI_TMP_DIR }} + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.CI_TMP_DIR }} + if: matrix.models == 'test' + - uses: actions/cache@v2 + id: node-modules-cache + with: + path: ~/.npm/ + key: node-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }} + - run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + if: matrix.models == 'test' + - name: Install STT package + run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + npm install --verbose ${{ env.CI_TMP_DIR }}/stt*.tgz + - run: | + ls -hal node_modules/stt* node_modules/.bin/ + - uses: ./.github/actions/run-tests + with: + runtime: "node" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + test-electronjs_all-Linux: + name: "Lin|Test MultiArchPlatform ElectronJS bindings" + runs-on: ubuntu-20.04 + needs: [repackage-nodejs-allplatforms, train-test-model-Linux] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + electronjs-version: [5.0.13, 12.0.0] + build-flavor: ["tf", "tflite"] + models: ["test", "prod"] + bitrate: ["8k", "16k"] + env: + CI_TMP_DIR: ${{ github.workspace }}/tmp/ + STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb + STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm + STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-node@v2 + with: + node-version: 12 + - run: | + sudo apt-get install -y --no-install-recommends sox + - uses: actions/download-artifact@v2 + with: + name: "stt-${{ matrix.build-flavor }}.tgz" + path: ${{ env.CI_TMP_DIR }} + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.CI_TMP_DIR }} + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + if: matrix.models == 'test' + - uses: actions/cache@v2 + id: electron-modules-cache + with: + path: ~/.npm/ + key: electron-modules-${{ matrix.build-flavor }}-${{ runner.os }}-${{ env.CI_NODE_MODULES_NTH }} + - name: Install STT package + run: | + ls -hal ${{ env.CI_TMP_DIR }}/ + npm install ${{ env.CI_TMP_DIR }}/stt*.tgz + - run: | + npm install electron@${{ matrix.electronjs-version }} + - uses: ./.github/actions/run-tests + with: + runtime: "electronjs" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + timeout-minutes: 5 test-nodejs_all-macOS: name: "Mac|Test MultiArchPlatform NodeJS bindings" runs-on: macos-10.15 @@ -1309,7 +1940,7 @@ jobs: test-nodejs_all-Windows: name: "Win|Test MultiArchPlatform NodeJS bindings" runs-on: windows-2019 - needs: [repackage-nodejs-allplatforms, train-test-model-macOS] + needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: matrix: @@ -1375,7 +2006,7 @@ jobs: test-electronjs_all-Windows: name: "Win|Test MultiArchPlatform ElectronJS bindings" runs-on: windows-2019 - needs: [repackage-nodejs-allplatforms, train-test-model-macOS] + needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: matrix: diff --git a/ci_scripts/all-vars.sh b/ci_scripts/all-vars.sh index 93db8c6b..8657b9dc 100755 --- a/ci_scripts/all-vars.sh +++ b/ci_scripts/all-vars.sh @@ -4,7 +4,7 @@ set -xe export OS=$(uname) if [ "${OS}" = "Linux" ]; then - export DS_ROOT_TASK=${HOME} + export DS_ROOT_TASK=${CI_TASK_DIR} export PYENV_ROOT="${DS_ROOT_TASK}/pyenv-root" export DS_CPU_COUNT=$(nproc) fi; diff --git a/ci_scripts/electronjs-tests-prod.sh b/ci_scripts/electronjs-tests-prod.sh index 1067d7f5..43c789a3 100755 --- a/ci_scripts/electronjs-tests-prod.sh +++ b/ci_scripts/electronjs-tests-prod.sh @@ -28,6 +28,12 @@ export_node_bin_path which electron which node +if [ "${OS}" = "Linux" ]; then + export DISPLAY=':99.0' + sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & + xvfb_process=$! +fi + node --version stt --version @@ -35,3 +41,8 @@ stt --version check_runtime_electronjs run_electronjs_prod_inference_tests "${bitrate}" + +if [ "${OS}" = "Linux" ]; then + sleep 1 + sudo kill -9 ${xvfb_process} || true +fi diff --git a/ci_scripts/electronjs-tests.sh b/ci_scripts/electronjs-tests.sh index 061f5720..368d4483 100755 --- a/ci_scripts/electronjs-tests.sh +++ b/ci_scripts/electronjs-tests.sh @@ -21,6 +21,12 @@ export_node_bin_path which electron which node +if [ "${OS}" = "Linux" ]; then + export DISPLAY=':99.0' + sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & + xvfb_process=$! +fi + node --version stt --version @@ -28,3 +34,8 @@ stt --version check_runtime_electronjs run_electronjs_inference_tests + +if [ "${OS}" = "Linux" ]; then + sleep 1 + sudo kill -9 ${xvfb_process} || true +fi diff --git a/ci_scripts/electronjs_tflite-tests-prod.sh b/ci_scripts/electronjs_tflite-tests-prod.sh index 63ff8be1..6d1a0b55 100755 --- a/ci_scripts/electronjs_tflite-tests-prod.sh +++ b/ci_scripts/electronjs_tflite-tests-prod.sh @@ -28,6 +28,12 @@ export_node_bin_path which electron which node +if [ "${OS}" = "Linux" ]; then + export DISPLAY=':99.0' + sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & + xvfb_process=$! +fi + node --version stt --version @@ -35,3 +41,8 @@ stt --version check_runtime_electronjs run_electronjs_prodtflite_inference_tests "${bitrate}" + +if [ "${OS}" = "Linux" ]; then + sleep 1 + sudo kill -9 ${xvfb_process} || true +fi diff --git a/ci_scripts/electronjs_tflite-tests.sh b/ci_scripts/electronjs_tflite-tests.sh index cd5f015b..ed5c3376 100755 --- a/ci_scripts/electronjs_tflite-tests.sh +++ b/ci_scripts/electronjs_tflite-tests.sh @@ -25,6 +25,12 @@ export_node_bin_path which electron which node +if [ "${OS}" = "Linux" ]; then + export DISPLAY=':99.0' + sudo Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & + xvfb_process=$! +fi + node --version stt --version @@ -32,3 +38,8 @@ stt --version check_runtime_electronjs run_electronjs_inference_tests + +if [ "${OS}" = "Linux" ]; then + sleep 1 + sudo kill -9 ${xvfb_process} || true +fi diff --git a/ci_scripts/tf-vars.sh b/ci_scripts/tf-vars.sh index 81dd0189..97da9fbc 100755 --- a/ci_scripts/tf-vars.sh +++ b/ci_scripts/tf-vars.sh @@ -4,7 +4,7 @@ set -ex export OS=$(uname) if [ "${OS}" = "Linux" ]; then - export DS_ROOT_TASK=$(/usr/bin/realpath "${HOME}") + export DS_ROOT_TASK=${CI_TASK_DIR} BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-linux-x86_64.sh BAZEL_SHA256=7ba815cbac712d061fe728fef958651512ff394b2708e89f79586ec93d1185ed