diff --git a/.github/actions/win-node-build/action.yml b/.github/actions/win-node-build/action.yml new file mode 100644 index 00000000..31352b04 --- /dev/null +++ b/.github/actions/win-node-build/action.yml @@ -0,0 +1,77 @@ +name: "NodeJS binding" +description: "Binding a nodejs binding" +inputs: + nodejs_versions: + description: "NodeJS versions supported" + required: true + electronjs_versions: + description: "ElectronJS versions supported" + required: true + local_cflags: + description: "CFLAGS for NodeJS package" + required: false + default: "" + local_ldflags: + description: "LDFLAGS for NodeJS package" + required: false + default: "" + local_libs: + description: "LIBS for NodeJS package" + required: false + default: "" + target: + description: "TARGET value" + required: false + default: "host" + chroot: + description: "RASPBIAN value" + required: false + default: "" +runs: + using: "composite" + steps: + - run: | + node --version + npm --version + shell: msys2 {0} + - run: | + npm update + shell: msys2 {0} + - run: | + mkdir -p tmp/headers/nodejs tmp/headers/electronjs + shell: msys2 {0} + - run: | + for node in ${{ inputs.nodejs_versions }}; do + EXTRA_CFLAGS=${{ inputs.local_cflags }} \ + EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \ + EXTRA_LIBS=${{ inputs.local_libs }} \ + make -C native_client/javascript \ + TARGET=${{ inputs.target }} \ + RASPBIAN=${{ inputs.chroot }} \ + NODE_ABI_TARGET=--target=${node} \ + NODE_DEVDIR=--devdir=headers/nodejs \ + clean node-wrapper + done; + shell: msys2 {0} + - run: | + for electron in ${{ inputs.electronjs_versions }}; do + EXTRA_CFLAGS=${{ inputs.local_cflags }} \ + EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \ + EXTRA_LIBS=${{ inputs.local_libs }} \ + make -C native_client/javascript \ + TARGET=${{ inputs.target }} \ + RASPBIAN=${{ inputs.chroot }} \ + NODE_ABI_TARGET=--target=${electron} \ + NODE_DIST_URL=--disturl=https://electronjs.org/headers \ + NODE_RUNTIME=--runtime=electron \ + NODE_DEVDIR=--devdir=headers/electronjs \ + clean node-wrapper + done; + shell: msys2 {0} + - run: | + make -C native_client/javascript clean npm-pack + shell: msys2 {0} + - run: | + tar -czf native_client/javascript/wrapper.tar.gz \ + -C native_client/javascript/ lib/ + shell: msys2 {0} diff --git a/.github/actions/win-numpy-vers/README.md b/.github/actions/win-numpy-vers/README.md new file mode 100644 index 00000000..57d28ece --- /dev/null +++ b/.github/actions/win-numpy-vers/README.md @@ -0,0 +1,14 @@ +GitHub Action to set NumPy versions +=================================== + +This actions aims at computing correct values for NumPy dependencies: + - `NUMPY_BUILD_VERSION`: range of accepted versions at Python binding build time + - `NUMPY_DEP_VERSION`: range of accepted versions for execution time + +Versions are set considering several factors: + - API and ABI compatibility ; otherwise we can have the binding wrapper + throwing errors like "Illegal instruction", or computing wrong values + because of changed memory layout + - Wheels availability: for CI and end users, we want to avoid having to + rebuild numpy so we stick to versions where there is an existing upstream + `wheel` file diff --git a/.github/actions/win-numpy-vers/action.yml b/.github/actions/win-numpy-vers/action.yml new file mode 100644 index 00000000..37f46082 --- /dev/null +++ b/.github/actions/win-numpy-vers/action.yml @@ -0,0 +1,93 @@ +name: "get numpy versions" +description: "Get proper NumPy build and runtime versions dependencies range" +inputs: + pyver: + description: "Python version" + required: true +outputs: + build_version: + description: "NumPy build dependency" + value: ${{ steps.numpy.outputs.build }} + dep_version: + description: "NumPy runtime dependency" + value: ${{ steps.numpy.outputs.dep }} +runs: + using: "composite" + steps: + - id: numpy + run: | + set -ex + NUMPY_BUILD_VERSION="==1.7.0" + NUMPY_DEP_VERSION=">=1.7.0" + + OS=$(uname -s) + ARCH=$(uname -m) + + case "${OS}:${ARCH}" in + Linux:x86_64) + case "${{ inputs.pyver }}" in + 3.7*) + NUMPY_BUILD_VERSION="==1.14.5" + NUMPY_DEP_VERSION=">=1.14.5,<=1.19.4" + ;; + 3.8*) + NUMPY_BUILD_VERSION="==1.17.3" + NUMPY_DEP_VERSION=">=1.17.3,<=1.19.4" + ;; + 3.9*) + NUMPY_BUILD_VERSION="==1.19.4" + NUMPY_DEP_VERSION=">=1.19.4,<=1.19.4" + ;; + esac + ;; + + Darwin:*) + case "${{ inputs.pyver }}" in + 3.6*) + NUMPY_BUILD_VERSION="==1.9.0" + NUMPY_DEP_VERSION=">=1.9.0" + ;; + 3.7*) + NUMPY_BUILD_VERSION="==1.14.5" + NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0" + ;; + 3.8*) + NUMPY_BUILD_VERSION="==1.17.3" + NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3" + ;; + 3.9*) + NUMPY_BUILD_VERSION="==1.19.4" + NUMPY_DEP_VERSION=">=1.19.4,<=1.19.4" + ;; + esac + ;; + + ${CI_MSYS_VERSION}:x86_64) + case "${{ inputs.pyver }}" in + 3.5*) + NUMPY_BUILD_VERSION="==1.11.0" + NUMPY_DEP_VERSION=">=1.11.0,<1.12.0" + ;; + 3.6*) + NUMPY_BUILD_VERSION="==1.12.0" + NUMPY_DEP_VERSION=">=1.12.0,<1.14.5" + ;; + 3.7*) + NUMPY_BUILD_VERSION="==1.14.5" + NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0" + ;; + 3.8*) + NUMPY_BUILD_VERSION="==1.17.3" + NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3" + ;; + 3.9*) + NUMPY_BUILD_VERSION="==1.19.4" + NUMPY_DEP_VERSION=">=1.19.4,<=1.19.4" + ;; + esac + ;; + esac + + echo "::set-output name=build::${NUMPY_BUILD_VERSION}" + echo "::set-output name=dep::${NUMPY_DEP_VERSION}" + shell: msys2 {0} diff --git a/.github/actions/win-python-build/action.yml b/.github/actions/win-python-build/action.yml new file mode 100644 index 00000000..cf7091c2 --- /dev/null +++ b/.github/actions/win-python-build/action.yml @@ -0,0 +1,31 @@ +name: "Python binding" +description: "Binding a python binding" +inputs: + numpy_build: + description: "NumPy build dependecy" + required: true + numpy_dep: + description: "NumPy runtime dependecy" + required: true +runs: + using: "composite" + steps: + - run: | + set -xe + + python3 --version + pip3 --version + + PROJECT_NAME="stt" + + NUMPY_BUILD_VERSION="${{ inputs.numpy_build }}" \ + NUMPY_DEP_VERSION="${{ inputs.numpy_dep }}" \ + EXTRA_CFLAGS=${{ inputs.local_cflags }} \ + EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \ + EXTRA_LIBS=${{ inputs.local_libs }} \ + make -C native_client/python/ \ + TARGET=${{ inputs.target }} \ + RASPBIAN=${{ inputs.chroot }} \ + SETUP_FLAGS="--project_name ${PROJECT_NAME}" \ + bindings-clean bindings + shell: msys2 {0} diff --git a/.github/actions/win-run-tests/action.yml b/.github/actions/win-run-tests/action.yml new file mode 100644 index 00000000..9529014b --- /dev/null +++ b/.github/actions/win-run-tests/action.yml @@ -0,0 +1,35 @@ +name: "Tests execution" +description: "Running tests" +inputs: + runtime: + description: "Runtime to use for running test" + required: true + model-kind: + description: "Running against CI baked or production model" + required: true + bitrate: + description: "Bitrate for testing" + required: true + chroot: + description: "Run using a chroot" + required: false +runs: + using: "composite" + steps: + - run: | + set -xe + + build="_tflite" + + model_kind="" + if [ "${{ inputs.model-kind }}" = "prod" ]; then + model_kind="-prod" + fi + + prefix="." + if [ ! -z "${{ inputs.chroot }}" ]; then + prefix="${{ inputs.chroot }}" + fi + + ${prefix}/ci_scripts/${{ inputs.runtime }}${build}-tests${model_kind}.sh ${{ inputs.bitrate }} + shell: msys2 {0} diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a7649af7..b06d17f1 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -18,7 +18,6 @@ env: # Windows specific CI_MSYS_VERSION: MSYS_NT-10.0-17763 - MSYS2_SHELL_PATH: D:\a\_temp\msys\msys64\usr\bin defaults: run: shell: bash @@ -1668,9 +1667,10 @@ jobs: name: "Win|Build CTC decoder Python package" needs: [swig_Windows_crosscompiled] runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -1693,15 +1693,22 @@ jobs: with: name: "swig_Windows_crosscompiled" path: ${{ github.workspace }}/native_client/ds-swig/ + - name: Remove /usr/bin/link conflicting with MSVC link.exe + run: | + set -ex + which link + rm /usr/bin/link + - name: Remove mingw32-make conflicting with MSYS make + run: | + set -ex + which mingw32-make + rm /c/ProgramData/Chocolatey/bin/mingw32-make - name: Link ds-swig into swig run: | set -ex ls -hal native_client/ds-swig/bin ln -s ds-swig.exe native_client/ds-swig/bin/swig.exe chmod +x native_client/ds-swig/bin/ds-swig.exe native_client/ds-swig/bin/swig.exe - - name: Remove /usr/bin/link conflicting with MSVC link.exe - run: | - rm /usr/bin/link - run: | make -C native_client/ctcdecode/ \ NUM_PROCESSES=$(nproc) \ @@ -1734,10 +1741,17 @@ jobs: name: "Win|Build TensorFlow (opt)" needs: tensorflow_opt-Windows runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} steps: - run: true + shell: bash if: needs.tensorflow_opt-Windows.outputs.status == 'found' - - uses: ilammy/msvc-dev-cmd@v1 + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: 'recursive' if: needs.tensorflow_opt-Windows.outputs.status == 'missing' - uses: msys2/setup-msys2@v2 with: @@ -1751,20 +1765,16 @@ jobs: unzip zip if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + - uses: ilammy/msvc-dev-cmd@v1 + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' - uses: actions/setup-python@v2 with: python-version: 3.7.9 if: needs.tensorflow_opt-Windows.outputs.status == 'missing' - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - submodules: 'recursive' - if: needs.tensorflow_opt-Windows.outputs.status == 'missing' - # It's important that this PATH change only happens *after* the checkout - # above, because otherwise the checkout fails when persisisting the - # credentials for submodules due to using MSYS2 Git - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH + - name: Workaround bazel bug when LLVM is installed https://github.com/bazelbuild/bazel/issues/12144 + run: | + rm -f /c/msys64/mingw64/clang-cl* + rm -rf "/c/Program Files/LLVM" if: needs.tensorflow_opt-Windows.outputs.status == 'missing' - run: ./ci_scripts/tf-setup.sh if: needs.tensorflow_opt-Windows.outputs.status == 'missing' @@ -1780,14 +1790,14 @@ jobs: build-lib_Windows: name: "Win|Build libstt+client" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [build-tensorflow-Windows, tensorflow_opt-Windows] steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: ilammy/msvc-dev-cmd@v1 - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -1800,6 +1810,7 @@ jobs: tar unzip zip + - uses: ilammy/msvc-dev-cmd@v1 - uses: ./.github/actions/check_artifact_exists with: name: ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz @@ -1808,8 +1819,10 @@ jobs: - run: | "C:/Program Files/7-Zip/7z.exe" x ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz -so | "C:/Program Files/7-Zip/7z.exe" x -aos -si -ttar -o`pwd` rm ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz - - run: | - git status + - name: Workaround bazel bug when LLVM is installed https://github.com/bazelbuild/bazel/issues/12144 + run: | + rm -f /c/msys64/mingw64/clang-cl* + rm -rf "/c/Program Files/LLVM" - run: ./ci_scripts/host-build.sh - run: ./ci_scripts/package.sh - uses: actions/upload-artifact@v2 @@ -1823,6 +1836,9 @@ jobs: build-python-Windows: name: "Win|Build Python bindings" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [build-lib_Windows, swig_Windows_crosscompiled] strategy: matrix: @@ -1830,8 +1846,6 @@ jobs: # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md python-version: [3.6.8, 3.7.9, 3.8.8, 3.9.4] steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -1871,13 +1885,16 @@ jobs: run: | rm /usr/bin/link - id: get_numpy - uses: ./.github/actions/numpy_vers + uses: ./.github/actions/win-numpy-vers with: pyver: ${{ matrix.python-version }} - - uses: ./.github/actions/python-build + - uses: ./.github/actions/win-python-build with: numpy_build: "${{ steps.get_numpy.outputs.build_version }}" numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}" + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: failure() - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-Windows.whl" @@ -1885,10 +1902,11 @@ jobs: build-nodejs-Windows: name: "Win|Build NodeJS/ElectronJS" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [build-lib_Windows, swig_Windows_crosscompiled] steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -1935,7 +1953,7 @@ jobs: with: path: native_client/javascript/headers/electronjs/ key: electron-headers-win-5.0.13_12.0.0 - - uses: ./.github/actions/node-build + - uses: ./.github/actions/win-node-build with: nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0 16.0.0" electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0" @@ -1950,17 +1968,19 @@ jobs: test-cpp-Windows: name: "Win|Test C++ binary" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [build-lib_Windows, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} env: CI_TMP_DIR: tmp/ STT_TEST_MODEL: tmp/output_graph.pb steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS + path-type: inherit update: true install: >- vim @@ -1986,7 +2006,7 @@ jobs: path: ${{ env.CI_TMP_DIR }} - run: | ls -hal ${{ env.CI_TMP_DIR }}/ - - uses: ./.github/actions/run-tests + - uses: ./.github/actions/win-run-tests with: runtime: "cppwin" bitrate: "16k" @@ -1994,6 +2014,9 @@ jobs: test-py-Windows: name: "Win|Test Python bindings" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [ build-python-Windows, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: @@ -2009,8 +2032,6 @@ jobs: STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: tmp/output_graph.pb steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -2040,7 +2061,7 @@ jobs: - run: | ls -hal ${{ env.CI_TMP_DIR }}/ python -m pip install --only-binary :all: --upgrade ${{ env.CI_TMP_DIR }}/stt*.whl - - uses: ./.github/actions/run-tests + - uses: ./.github/actions/win-run-tests with: runtime: "python" bitrate: ${{ matrix.bitrate }} @@ -2048,6 +2069,9 @@ jobs: test-nodejs-Windows: name: "Win|Test NodeJS bindings" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [ build-nodejs-Windows, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: @@ -2062,8 +2086,6 @@ jobs: STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: tmp/output_graph.pb steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -2103,7 +2125,7 @@ jobs: run: | ls -hal ${{ env.CI_TMP_DIR }}/ npm install ${{ env.CI_TMP_DIR }}/stt*.tgz - - uses: ./.github/actions/run-tests + - uses: ./.github/actions/win-run-tests with: runtime: "node" bitrate: ${{ matrix.bitrate }} @@ -2111,6 +2133,9 @@ jobs: test-electronjs-Windows: name: "Win|Test ElectronJS bindings" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [ build-nodejs-Windows, train-test-model-Linux ] if: ${{ github.event_name == 'pull_request' }} strategy: @@ -2124,8 +2149,6 @@ jobs: STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: tmp/output_graph.pb steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -2167,7 +2190,7 @@ jobs: npm install ${{ env.CI_TMP_DIR }}/stt*.tgz - run: | npm install electron@${{ matrix.electronjs-version }} - - uses: ./.github/actions/run-tests + - uses: ./.github/actions/win-run-tests with: runtime: "electronjs" bitrate: ${{ matrix.bitrate }} @@ -2433,6 +2456,9 @@ jobs: test-nodejs_all-Windows: name: "Win|Test MultiArchPlatform NodeJS bindings" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: @@ -2448,8 +2474,6 @@ jobs: STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: tmp/output_graph.pb steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -2489,7 +2513,7 @@ jobs: run: | ls -hal ${{ env.CI_TMP_DIR }}/ npm install --verbose ${{ env.CI_TMP_DIR }}/stt*.tgz - - uses: ./.github/actions/run-tests + - uses: ./.github/actions/win-run-tests with: runtime: "node" bitrate: ${{ matrix.bitrate }} @@ -2497,6 +2521,9 @@ jobs: test-electronjs_all-Windows: name: "Win|Test MultiArchPlatform ElectronJS bindings" runs-on: windows-2019 + defaults: + run: + shell: msys2 {0} needs: [repackage-nodejs-allplatforms, train-test-model-Linux] if: ${{ github.event_name == 'pull_request' }} strategy: @@ -2511,8 +2538,6 @@ jobs: STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm STT_TEST_MODEL: tmp/output_graph.pb steps: - - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front - run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH - uses: msys2/setup-msys2@v2 with: msystem: MSYS @@ -2554,7 +2579,7 @@ jobs: npm install --verbose ${{ env.CI_TMP_DIR }}/stt*.tgz - run: | npm install electron@${{ matrix.electronjs-version }} - - uses: ./.github/actions/run-tests + - uses: ./.github/actions/win-run-tests with: runtime: "electronjs" bitrate: ${{ matrix.bitrate }} diff --git a/ci_scripts/tf-vars.sh b/ci_scripts/tf-vars.sh index 14022e91..be38ecf3 100755 --- a/ci_scripts/tf-vars.sh +++ b/ci_scripts/tf-vars.sh @@ -31,7 +31,7 @@ elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then export DS_ROOT_TASK=${CI_TASK_DIR} export BAZEL_VC="C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC" - # export BAZEL_VC_FULL_VERSION="14.28.30037" + export BAZEL_VC_FULL_VERSION="14.29.30133" export MSYS2_ARG_CONV_EXCL='//' mkdir -p ${CI_TASK_DIR}/tmp/