experiments
/
STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix build on Windows after internal GitHub Actions MSYS2 changes

This commit is contained in:
Reuben Morais 2021-10-27 10:48:28 +02:00
parent 83b40b2532
commit d753431d11
7 changed files with 323 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
.github/actions/win-node-build/action.yml vendored Normal file
View File

@ -0,0 +1,77 @@
name: "NodeJS binding"
description: "Binding a nodejs binding"
inputs:
nodejs_versions:
description: "NodeJS versions supported"
required: true
electronjs_versions:
description: "ElectronJS versions supported"
required: true
local_cflags:
description: "CFLAGS for NodeJS package"
required: false
default: ""
local_ldflags:
description: "LDFLAGS for NodeJS package"
required: false
default: ""
local_libs:
description: "LIBS for NodeJS package"
required: false
default: ""
target:
description: "TARGET value"
required: false
default: "host"
chroot:
description: "RASPBIAN value"
required: false
default: ""
runs:
using: "composite"
steps:
- run: |
node --version
npm --version
shell: msys2 {0}
- run: |
npm update
shell: msys2 {0}
- run: |
mkdir -p tmp/headers/nodejs tmp/headers/electronjs
shell: msys2 {0}
- run: |
for node in ${{ inputs.nodejs_versions }}; do
EXTRA_CFLAGS=${{ inputs.local_cflags }} \
EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \
EXTRA_LIBS=${{ inputs.local_libs }} \
make -C native_client/javascript \
TARGET=${{ inputs.target }} \
RASPBIAN=${{ inputs.chroot }} \
NODE_ABI_TARGET=--target=${node} \
NODE_DEVDIR=--devdir=headers/nodejs \
clean node-wrapper
done;
shell: msys2 {0}
- run: |
for electron in ${{ inputs.electronjs_versions }}; do
EXTRA_CFLAGS=${{ inputs.local_cflags }} \
EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \
EXTRA_LIBS=${{ inputs.local_libs }} \
make -C native_client/javascript \
TARGET=${{ inputs.target }} \
RASPBIAN=${{ inputs.chroot }} \
NODE_ABI_TARGET=--target=${electron} \
NODE_DIST_URL=--disturl=https://electronjs.org/headers \
NODE_RUNTIME=--runtime=electron \
NODE_DEVDIR=--devdir=headers/electronjs \
clean node-wrapper
done;
shell: msys2 {0}
- run: |
make -C native_client/javascript clean npm-pack
shell: msys2 {0}
- run: |
tar -czf native_client/javascript/wrapper.tar.gz \
-C native_client/javascript/ lib/
shell: msys2 {0}

14
.github/actions/win-numpy-vers/README.md vendored Normal file
View File

@ -0,0 +1,14 @@
GitHub Action to set NumPy versions
===================================
This actions aims at computing correct values for NumPy dependencies:
- `NUMPY_BUILD_VERSION`: range of accepted versions at Python binding build time
- `NUMPY_DEP_VERSION`: range of accepted versions for execution time
Versions are set considering several factors:
- API and ABI compatibility ; otherwise we can have the binding wrapper
throwing errors like "Illegal instruction", or computing wrong values
because of changed memory layout
- Wheels availability: for CI and end users, we want to avoid having to
rebuild numpy so we stick to versions where there is an existing upstream
`wheel` file

93
.github/actions/win-numpy-vers/action.yml vendored Normal file
View File

@ -0,0 +1,93 @@
name: "get numpy versions"
description: "Get proper NumPy build and runtime versions dependencies range"
inputs:
pyver:
description: "Python version"
required: true
outputs:
build_version:
description: "NumPy build dependency"
value: ${{ steps.numpy.outputs.build }}
dep_version:
description: "NumPy runtime dependency"
value: ${{ steps.numpy.outputs.dep }}
runs:
using: "composite"
steps:
- id: numpy
run: |
set -ex
NUMPY_BUILD_VERSION="==1.7.0"
NUMPY_DEP_VERSION=">=1.7.0"
OS=$(uname -s)
ARCH=$(uname -m)
case "${OS}:${ARCH}" in
Linux:x86_64)
case "${{ inputs.pyver }}" in
3.7*)
NUMPY_BUILD_VERSION="==1.14.5"
NUMPY_DEP_VERSION=">=1.14.5,<=1.19.4"
;;
3.8*)
NUMPY_BUILD_VERSION="==1.17.3"
NUMPY_DEP_VERSION=">=1.17.3,<=1.19.4"
;;
3.9*)
NUMPY_BUILD_VERSION="==1.19.4"
NUMPY_DEP_VERSION=">=1.19.4,<=1.19.4"
;;
esac
;;
Darwin:*)
case "${{ inputs.pyver }}" in
3.6*)
NUMPY_BUILD_VERSION="==1.9.0"
NUMPY_DEP_VERSION=">=1.9.0"
;;
3.7*)
NUMPY_BUILD_VERSION="==1.14.5"
NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0"
;;
3.8*)
NUMPY_BUILD_VERSION="==1.17.3"
NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3"
;;
3.9*)
NUMPY_BUILD_VERSION="==1.19.4"
NUMPY_DEP_VERSION=">=1.19.4,<=1.19.4"
;;
esac
;;
${CI_MSYS_VERSION}:x86_64)
case "${{ inputs.pyver }}" in
3.5*)
NUMPY_BUILD_VERSION="==1.11.0"
NUMPY_DEP_VERSION=">=1.11.0,<1.12.0"
;;
3.6*)
NUMPY_BUILD_VERSION="==1.12.0"
NUMPY_DEP_VERSION=">=1.12.0,<1.14.5"
;;
3.7*)
NUMPY_BUILD_VERSION="==1.14.5"
NUMPY_DEP_VERSION=">=1.14.5,<=1.17.0"
;;
3.8*)
NUMPY_BUILD_VERSION="==1.17.3"
NUMPY_DEP_VERSION=">=1.17.3,<=1.17.3"
;;
3.9*)
NUMPY_BUILD_VERSION="==1.19.4"
NUMPY_DEP_VERSION=">=1.19.4,<=1.19.4"
;;
esac
;;
esac
echo "::set-output name=build::${NUMPY_BUILD_VERSION}"
echo "::set-output name=dep::${NUMPY_DEP_VERSION}"
shell: msys2 {0}

31
.github/actions/win-python-build/action.yml vendored Normal file
View File

@ -0,0 +1,31 @@
name: "Python binding"
description: "Binding a python binding"
inputs:
numpy_build:
description: "NumPy build dependecy"
required: true
numpy_dep:
description: "NumPy runtime dependecy"
required: true
runs:
using: "composite"
steps:
- run: |
set -xe
python3 --version
pip3 --version
PROJECT_NAME="stt"
NUMPY_BUILD_VERSION="${{ inputs.numpy_build }}" \
NUMPY_DEP_VERSION="${{ inputs.numpy_dep }}" \
EXTRA_CFLAGS=${{ inputs.local_cflags }} \
EXTRA_LDFLAGS=${{ inputs.local_ldflags }} \
EXTRA_LIBS=${{ inputs.local_libs }} \
make -C native_client/python/ \
TARGET=${{ inputs.target }} \
RASPBIAN=${{ inputs.chroot }} \
SETUP_FLAGS="--project_name ${PROJECT_NAME}" \
bindings-clean bindings
shell: msys2 {0}

35
.github/actions/win-run-tests/action.yml vendored Normal file
View File

@ -0,0 +1,35 @@
name: "Tests execution"
description: "Running tests"
inputs:
runtime:
description: "Runtime to use for running test"
required: true
model-kind:
description: "Running against CI baked or production model"
required: true
bitrate:
description: "Bitrate for testing"
required: true
chroot:
description: "Run using a chroot"
required: false
runs:
using: "composite"
steps:
- run: |
set -xe
build="_tflite"
model_kind=""
if [ "${{ inputs.model-kind }}" = "prod" ]; then
model_kind="-prod"
fi
prefix="."
if [ ! -z "${{ inputs.chroot }}" ]; then
prefix="${{ inputs.chroot }}"
fi
${prefix}/ci_scripts/${{ inputs.runtime }}${build}-tests${model_kind}.sh ${{ inputs.bitrate }}
shell: msys2 {0}

119
.github/workflows/build-and-test.yml vendored
View File

@ -18,7 +18,6 @@ env:
# Windows specific
CI_MSYS_VERSION: MSYS_NT-10.0-17763
MSYS2_SHELL_PATH: D:\a\_temp\msys\msys64\usr\bin
defaults:
run:
shell: bash
@ -1668,9 +1667,10 @@ jobs:
name: "Win|Build CTC decoder Python package"
needs: [swig_Windows_crosscompiled]
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -1693,15 +1693,22 @@ jobs:
with:
name: "swig_Windows_crosscompiled"
path: ${{ github.workspace }}/native_client/ds-swig/
- name: Remove /usr/bin/link conflicting with MSVC link.exe
run: |
set -ex
which link
rm /usr/bin/link
- name: Remove mingw32-make conflicting with MSYS make
run: |
set -ex
which mingw32-make
rm /c/ProgramData/Chocolatey/bin/mingw32-make
- name: Link ds-swig into swig
run: |
set -ex
ls -hal native_client/ds-swig/bin
ln -s ds-swig.exe native_client/ds-swig/bin/swig.exe
chmod +x native_client/ds-swig/bin/ds-swig.exe native_client/ds-swig/bin/swig.exe
- name: Remove /usr/bin/link conflicting with MSVC link.exe
run: |
rm /usr/bin/link
- run: |
make -C native_client/ctcdecode/ \
NUM_PROCESSES=$(nproc) \
@ -1734,10 +1741,17 @@ jobs:
name: "Win|Build TensorFlow (opt)"
needs: tensorflow_opt-Windows
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
steps:
- run: true
shell: bash
if: needs.tensorflow_opt-Windows.outputs.status == 'found'
- uses: ilammy/msvc-dev-cmd@v1
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
- uses: msys2/setup-msys2@v2
with:
@ -1751,20 +1765,16 @@ jobs:
unzip
zip
if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
- uses: ilammy/msvc-dev-cmd@v1
if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
- uses: actions/setup-python@v2
with:
python-version: 3.7.9
if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
# It's important that this PATH change only happens *after* the checkout
# above, because otherwise the checkout fails when persisisting the
# credentials for submodules due to using MSYS2 Git
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- name: Workaround bazel bug when LLVM is installed https://github.com/bazelbuild/bazel/issues/12144
run: |
rm -f /c/msys64/mingw64/clang-cl*
rm -rf "/c/Program Files/LLVM"
if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
- run: ./ci_scripts/tf-setup.sh
if: needs.tensorflow_opt-Windows.outputs.status == 'missing'
@ -1780,14 +1790,14 @@ jobs:
build-lib_Windows:
name: "Win|Build libstt+client"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [build-tensorflow-Windows, tensorflow_opt-Windows]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: ilammy/msvc-dev-cmd@v1
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -1800,6 +1810,7 @@ jobs:
tar
unzip
zip
- uses: ilammy/msvc-dev-cmd@v1
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz
@ -1808,8 +1819,10 @@ jobs:
- run: |
"C:/Program Files/7-Zip/7z.exe" x ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz -so | "C:/Program Files/7-Zip/7z.exe" x -aos -si -ttar -o`pwd`
rm ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz
- run: |
git status
- name: Workaround bazel bug when LLVM is installed https://github.com/bazelbuild/bazel/issues/12144
run: |
rm -f /c/msys64/mingw64/clang-cl*
rm -rf "/c/Program Files/LLVM"
- run: ./ci_scripts/host-build.sh
- run: ./ci_scripts/package.sh
- uses: actions/upload-artifact@v2
@ -1823,6 +1836,9 @@ jobs:
build-python-Windows:
name: "Win|Build Python bindings"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [build-lib_Windows, swig_Windows_crosscompiled]
strategy:
matrix:
@ -1830,8 +1846,6 @@ jobs:
# https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md
python-version: [3.6.8, 3.7.9, 3.8.8, 3.9.4]
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -1871,13 +1885,16 @@ jobs:
run: |
rm /usr/bin/link
- id: get_numpy
uses: ./.github/actions/numpy_vers
uses: ./.github/actions/win-numpy-vers
with:
pyver: ${{ matrix.python-version }}
- uses: ./.github/actions/python-build
- uses: ./.github/actions/win-python-build
with:
numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}"
- name: Setup tmate session
uses: mxschmitt/action-tmate@v3
if: failure()
- uses: actions/upload-artifact@v2
with:
name: "stt-tflite-${{ matrix.python-version }}-Windows.whl"
@ -1885,10 +1902,11 @@ jobs:
build-nodejs-Windows:
name: "Win|Build NodeJS/ElectronJS"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [build-lib_Windows, swig_Windows_crosscompiled]
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -1935,7 +1953,7 @@ jobs:
with:
path: native_client/javascript/headers/electronjs/
key: electron-headers-win-5.0.13_12.0.0
- uses: ./.github/actions/node-build
- uses: ./.github/actions/win-node-build
with:
nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0 16.0.0"
electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0"
@ -1950,17 +1968,19 @@ jobs:
test-cpp-Windows:
name: "Win|Test C++ binary"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [build-lib_Windows, train-test-model-Linux]
if: ${{ github.event_name == 'pull_request' }}
env:
CI_TMP_DIR: tmp/
STT_TEST_MODEL: tmp/output_graph.pb
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
path-type: inherit
update: true
install: >-
vim
@ -1986,7 +2006,7 @@ jobs:
path: ${{ env.CI_TMP_DIR }}
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
- uses: ./.github/actions/run-tests
- uses: ./.github/actions/win-run-tests
with:
runtime: "cppwin"
bitrate: "16k"
@ -1994,6 +2014,9 @@ jobs:
test-py-Windows:
name: "Win|Test Python bindings"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [ build-python-Windows, train-test-model-Linux ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
@ -2009,8 +2032,6 @@ jobs:
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -2040,7 +2061,7 @@ jobs:
- run: |
ls -hal ${{ env.CI_TMP_DIR }}/
python -m pip install --only-binary :all: --upgrade ${{ env.CI_TMP_DIR }}/stt*.whl
- uses: ./.github/actions/run-tests
- uses: ./.github/actions/win-run-tests
with:
runtime: "python"
bitrate: ${{ matrix.bitrate }}
@ -2048,6 +2069,9 @@ jobs:
test-nodejs-Windows:
name: "Win|Test NodeJS bindings"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [ build-nodejs-Windows, train-test-model-Linux ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
@ -2062,8 +2086,6 @@ jobs:
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -2103,7 +2125,7 @@ jobs:
run: |
ls -hal ${{ env.CI_TMP_DIR }}/
npm install ${{ env.CI_TMP_DIR }}/stt*.tgz
- uses: ./.github/actions/run-tests
- uses: ./.github/actions/win-run-tests
with:
runtime: "node"
bitrate: ${{ matrix.bitrate }}
@ -2111,6 +2133,9 @@ jobs:
test-electronjs-Windows:
name: "Win|Test ElectronJS bindings"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [ build-nodejs-Windows, train-test-model-Linux ]
if: ${{ github.event_name == 'pull_request' }}
strategy:
@ -2124,8 +2149,6 @@ jobs:
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -2167,7 +2190,7 @@ jobs:
npm install ${{ env.CI_TMP_DIR }}/stt*.tgz
- run: |
npm install electron@${{ matrix.electronjs-version }}
- uses: ./.github/actions/run-tests
- uses: ./.github/actions/win-run-tests
with:
runtime: "electronjs"
bitrate: ${{ matrix.bitrate }}
@ -2433,6 +2456,9 @@ jobs:
test-nodejs_all-Windows:
name: "Win|Test MultiArchPlatform NodeJS bindings"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
if: ${{ github.event_name == 'pull_request' }}
strategy:
@ -2448,8 +2474,6 @@ jobs:
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -2489,7 +2513,7 @@ jobs:
run: |
ls -hal ${{ env.CI_TMP_DIR }}/
npm install --verbose ${{ env.CI_TMP_DIR }}/stt*.tgz
- uses: ./.github/actions/run-tests
- uses: ./.github/actions/win-run-tests
with:
runtime: "node"
bitrate: ${{ matrix.bitrate }}
@ -2497,6 +2521,9 @@ jobs:
test-electronjs_all-Windows:
name: "Win|Test MultiArchPlatform ElectronJS bindings"
runs-on: windows-2019
defaults:
run:
shell: msys2 {0}
needs: [repackage-nodejs-allplatforms, train-test-model-Linux]
if: ${{ github.event_name == 'pull_request' }}
strategy:
@ -2511,8 +2538,6 @@ jobs:
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- uses: msys2/setup-msys2@v2
with:
msystem: MSYS
@ -2554,7 +2579,7 @@ jobs:
npm install --verbose ${{ env.CI_TMP_DIR }}/stt*.tgz
- run: |
npm install electron@${{ matrix.electronjs-version }}
- uses: ./.github/actions/run-tests
- uses: ./.github/actions/win-run-tests
with:
runtime: "electronjs"
bitrate: ${{ matrix.bitrate }}

2
ci_scripts/tf-vars.sh
View File

@ -31,7 +31,7 @@ elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
export DS_ROOT_TASK=${CI_TASK_DIR}
export BAZEL_VC="C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC"
# export BAZEL_VC_FULL_VERSION="14.28.30037"
export BAZEL_VC_FULL_VERSION="14.29.30133"
export MSYS2_ARG_CONV_EXCL='//'
mkdir -p ${CI_TASK_DIR}/tmp/