From 7855f0a5637551788014d729c00c3c46d8d34223 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 30 Mar 2021 12:33:32 +0200 Subject: [PATCH] Base Windows CI setup --- .github/actions/numpy_vers/action.yml | 3 +- .github/actions/win-install-sox/action.yml | 12 + .github/workflows/windows-amd64.yml | 653 +++++++++++++++++++++ ci_scripts/all-utils.sh | 9 +- ci_scripts/all-vars.sh | 1 + ci_scripts/cppwin_tflite-tests.sh | 23 + ci_scripts/tf-build.sh | 6 +- ci_scripts/tf-setup.sh | 2 +- ci_scripts/tf-vars.sh | 32 +- native_client/ctcdecode/Makefile | 12 +- native_client/definitions.mk | 2 +- 11 files changed, 721 insertions(+), 34 deletions(-) create mode 100644 .github/actions/win-install-sox/action.yml create mode 100644 .github/workflows/windows-amd64.yml create mode 100755 ci_scripts/cppwin_tflite-tests.sh diff --git a/.github/actions/numpy_vers/action.yml b/.github/actions/numpy_vers/action.yml index 9671df81..607499cc 100644 --- a/.github/actions/numpy_vers/action.yml +++ b/.github/actions/numpy_vers/action.yml @@ -16,6 +16,7 @@ runs: steps: - id: numpy run: | + set -ex NUMPY_BUILD_VERSION="==1.7.0" NUMPY_DEP_VERSION=">=1.7.0" @@ -62,7 +63,7 @@ runs: ;; # TODO: 'Windows*' might not be good - Windows*:x86_64) + MSYS_NT-10.0-17763:x86_64) case "${{ inputs.pyver }}" in 3.5*) NUMPY_BUILD_VERSION="==1.11.0" diff --git a/.github/actions/win-install-sox/action.yml b/.github/actions/win-install-sox/action.yml new file mode 100644 index 00000000..81ebdbd7 --- /dev/null +++ b/.github/actions/win-install-sox/action.yml @@ -0,0 +1,12 @@ +name: "Install SoX and add to PATH" +description: "Install SoX and add to PATH" +runs: + using: "composite" + steps: + - run: | + set -ex + wget https://sourceforge.net/projects/sox/files/sox/14.4.2/sox-14.4.2-win32.zip/download -O sox-14.4.2-win32.zip + "C:/Program Files/7-Zip/7z.exe" x -o`pwd`/bin/ -tzip -aoa sox-14.4.2-win32.zip + rm sox-*zip + echo "`pwd`/bin/sox-14.4.2/" >> $GITHUB_PATH + shell: bash diff --git a/.github/workflows/windows-amd64.yml b/.github/workflows/windows-amd64.yml new file mode 100644 index 00000000..25a9099e --- /dev/null +++ b/.github/workflows/windows-amd64.yml @@ -0,0 +1,653 @@ +name: "Windows amd64" +on: + pull_request: + push: + branches: + - master +env: + TASKCLUSTER_TASK_DIR: ${{ github.workspace }} + TASKCLUSTER_ARTIFACTS: ${{ github.workspace }}/artifacts + TC_MSYS_VERSION: MSYS_NT-10.0-17763 +defaults: + run: + shell: bash +jobs: + swig_Windows: + name: "Build SWIG for Windows" + runs-on: windows-2019 + steps: + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + update: true + install: >- + autoconf + automake + bison + gcc + git + make + pcre-devel + - uses: actions/checkout@v2 + with: + repository: "swig/swig" + ref: "90cdbee6a69d13b39d734083b9f91069533b0d7b" + - run: | + mkdir -p build-static/ + - run: | + sh autogen.sh + ./configure \ + --prefix=`pwd`/build-static/ \ + --program-prefix=ds- + - run: | + make -j + - run: | + make install + - uses: actions/upload-artifact@v2 + with: + name: ${{ github.job }} + path: ${{ github.workspace }}/build-static/ + swig_Linux: + name: "Build SWIG for Linux" + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + repository: "swig/swig" + ref: "90cdbee6a69d13b39d734083b9f91069533b0d7b" + - run: | + sudo apt-get install -y --no-install-recommends autoconf automake build-essential bison libpcre2-dev + - run: | + mkdir -p build-static/ + - run: | + sh autogen.sh + ./configure \ + --prefix=${{ github.workspace }}/build-static/ \ + --program-prefix=ds- + - run: | + make -j + - run: | + make install + - uses: actions/upload-artifact@v2 + with: + name: ${{ github.job }} + path: ${{ github.workspace }}/build-static/ + build-ctc-decoder-windows: + name: "Build CTC decoder Windows Python package for testing" + needs: [swig_Windows] + runs-on: windows-2019 + steps: + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + path-type: inherit + update: true + install: >- + git + make + - uses: ilammy/msvc-dev-cmd@v1 + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: 3.7.9 + - run: | + python --version + python -m pip --version + - uses: actions/download-artifact@v2 + with: + name: "swig_Windows" + path: ${{ github.workspace }}/native_client/ds-swig/ + - name: Link ds-swig into swig + run: | + set -ex + ls -hal native_client/ds-swig/bin + ln -s ds-swig.exe native_client/ds-swig/bin/swig.exe + chmod +x native_client/ds-swig/bin/ds-swig.exe native_client/ds-swig/bin/swig.exe + - name: Remove /usr/bin/link conflicting with MSVC link.exe + run: | + rm /usr/bin/link + - run: | + make -C native_client/ctcdecode/ \ + NUM_PROCESSES=$(nproc) \ + bindings + - uses: actions/upload-artifact@v2 + with: + name: "ds_ctcdecoder-windows-test.whl" + path: ${{ github.workspace }}/native_client/ctcdecode/dist/*.whl + - run: | + make -C native_client/ctcdecode clean-keep-third-party + build-ctc-decoder-linux: + name: "Build CTC decoder Linux Python package for testing" + needs: [swig_Linux] + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + - run: | + python --version + pip --version + - uses: actions/download-artifact@v2 + with: + name: "swig_Linux" + path: ${{ github.workspace }}/native_client/ds-swig/ + - run: | + ls -hal ${{ github.workspace }}/native_client/ds-swig/bin + ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig + chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig + - run: | + make -C native_client/ctcdecode/ \ + NUM_PROCESSES=$(nproc) \ + bindings + - uses: actions/upload-artifact@v2 + with: + name: "ds_ctcdecoder-linux-test.whl" + path: ${{ github.workspace }}/native_client/ctcdecode/dist/*.whl + - run: | + make -C native_client/ctcdecode clean-keep-third-party + train-test-model: + name: "Train a test model" + needs: ["build-ctc-decoder-linux"] + runs-on: ubuntu-20.04 + strategy: + matrix: + build-flavor: ["tf", "tflite"] + bitrate: ["8k", "16k"] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v2 + with: + python-version: 3.6 + - uses: actions/download-artifact@v2 + with: + name: "ds_ctcdecoder-linux-test.whl" + - run: | + python --version + pip --version + - run: | + pip install --upgrade pip==19.3.1 setuptools==45.0.0 wheel==0.33.6 + - run: | + pip install ds_ctcdecoder-*-cp36-cp36m-manylinux1_x86_64.whl + DS_NODECODER=y pip install --upgrade . + - run: | + bits="" + if [ "${{ matrix.bitrate }}" = "8k" ]; then + bits=8000 + fi + if [ "${{ matrix.bitrate }}" = "16k" ]; then + bits=16000 + fi + + # Easier to rename to that we can exercize the LDC93S1 importer code to + # generate the CSV file. + echo "Moving ${bits} to LDC93S1.wav" + mv data/smoke_test/LDC93S1_pcms16le_1_${bits}.wav data/smoke_test/LDC93S1.wav + + ./bin/run-tc-ldc93s1_new.sh 249 ${bits} + if [ "${{ matrix.build-flavor }}" = "tflite" ]; then + ./bin/run-tc-ldc93s1_tflite.sh ${bits} + fi + - run: | + curl -vsSL https://github.com/mozilla/DeepSpeech/releases/download/v0.9.3/linux.amd64.convert_graphdef_memmapped_format.xz | xz -d > /tmp/convert_graphdef_memmapped_format + chmod +x /tmp/convert_graphdef_memmapped_format + /tmp/convert_graphdef_memmapped_format --in_graph=/tmp/train/output_graph.pb --out_graph=/tmp/train/output_graph.pbmm + if: matrix.build-flavor == 'tf' + - run: | + cp /tmp/train*/output_graph.* /tmp/ + - run: | + tar -cf - \ + -C /tmp/ckpt/ . \ + | xz -9 -T0 > /tmp/checkpoint.tar.xz + - uses: actions/upload-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: /tmp/output_graph.* + - uses: actions/upload-artifact@v2 + with: + name: "test-checkpoint.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: /tmp/checkpoint.tar.xz + tensorflow_opt-Windows: + name: "Check cache for TensorFlow" + runs-on: ubuntu-20.04 + outputs: + status: ${{ steps.check_artifact_exists.outputs.status }} + cache_key: ${{ steps.get_cache_key.outputs.key }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - id: get_cache_key + uses: ./.github/actions/get_cache_key + with: + extras: "7" + - id: check_artifact_exists + uses: ./.github/actions/check_artifact_exists + with: + name: ${{ steps.get_cache_key.outputs.key }} + build-tensorflow-Windows: + name: "Build TensorFlow (opt) for Windows" + needs: tensorflow_opt-Windows + runs-on: windows-2019 + steps: + - run: true + if: needs.tensorflow_opt-Windows.outputs.status == 'found' + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + path-type: inherit + update: true + install: >- + git + patch + tar + unzip + zip + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + - uses: actions/setup-python@v2 + with: + python-version: 3.7.9 + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: 'recursive' + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + # It's important that this PATH change only happens *after* the checkout + # above, because otherwise the checkout fails when persisisting the + # credentials for submodules due to using MSYS2 Git + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + - run: ./ci_scripts/tf-setup.sh + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + - run: ./ci_scripts/tf-build.sh "--windows-cpu" + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + - run: ./ci_scripts/tf-package.sh + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + - uses: actions/upload-artifact@v2 + with: + name: ${{ needs.tensorflow_opt-Windows.outputs.cache_key }} + path: ${{ github.workspace }}/artifacts/home.tar.xz + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + build-lib_Windows: + name: "Build libdeepspeech.so and deepspeech C++ binary" + runs-on: windows-2019 + needs: [build-tensorflow-Windows, tensorflow_opt-Windows] + strategy: + matrix: + build-flavor: ["tf", "tflite"] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: ilammy/msvc-dev-cmd@v1 + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + update: true + install: >- + git + make + patch + pkg-config + tar + unzip + zip + - uses: actions/download-artifact@v2 + with: + name: ${{ needs.tensorflow_opt-Windows.outputs.cache_key }} + path: ${{ github.workspace }}/ + if: needs.tensorflow_opt-Windows.outputs.status == 'missing' + - uses: ./.github/actions/check_artifact_exists + with: + name: ${{ needs.tensorflow_opt-Windows.outputs.cache_key }} + path: ${{ github.workspace }}/ + download: true + if: needs.tensorflow_opt-Windows.outputs.status == 'found' + - run: | + "C:/Program Files/7-Zip/7z.exe" x home.tar.xz -so | "C:/Program Files/7-Zip/7z.exe" x -aos -si -ttar -o`pwd` + rm home.tar.xz + - run: | + git status + - run: ./ci_scripts/host-build.sh ${{ matrix.build-flavor }} + - run: ./ci_scripts/package.sh + - uses: actions/upload-artifact@v2 + with: + name: "native_client.${{ matrix.build-flavor }}.tar.xz" + path: ${{ github.workspace }}/artifacts/native_client.tar.xz + - uses: actions/upload-artifact@v2 + with: + name: "libdeepspeech.${{ matrix.build-flavor }}.zip" + path: ${{ github.workspace }}/artifacts/libdeepspeech.zip + build-python-Windows: + name: "Build Python bindings for Windows" + runs-on: windows-2019 + needs: [build-lib_Windows, swig_Windows] + strategy: + matrix: + build-flavor: ["tf", "tflite"] + python-version: [3.6.8, 3.7.9, 3.8.8, 3.9.2] + steps: + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + path-type: inherit + update: true + install: >- + make + - uses: ilammy/msvc-dev-cmd@v1 + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/download-artifact@v2 + with: + name: "native_client.${{ matrix.build-flavor }}.tar.xz" + path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ + - run: | + pushd tensorflow/bazel-bin/native_client/ + "C:/Program Files/7-Zip/7z.exe" x native_client.tar.xz -so | "C:/Program Files/7-Zip/7z.exe" x -aoa -si -ttar -o`pwd` + ls -hal + popd + - uses: actions/download-artifact@v2 + with: + name: "swig_Windows" + path: ${{ github.workspace }}/native_client/ds-swig/ + - name: Link ds-swig into swig + run: | + set -ex + ls -hal native_client/ds-swig/bin + ln -s ds-swig.exe native_client/ds-swig/bin/swig.exe + chmod +x native_client/ds-swig/bin/ds-swig.exe native_client/ds-swig/bin/swig.exe + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Remove /usr/bin/link conflicting with MSVC link.exe + run: | + rm /usr/bin/link + - id: get_numpy + uses: ./.github/actions/numpy_vers + with: + pyver: ${{ matrix.python-version }} + - uses: ./.github/actions/python-build + with: + build_flavor: ${{ matrix.build-flavor }} + numpy_build: "${{ steps.get_numpy.outputs.build_version }}" + numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}" + - uses: actions/upload-artifact@v2 + with: + name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl" + path: ${{ github.workspace }}/wheels/*.whl + build-nodejs-Windows: + name: "Build NodeJS/ElectronJS for Windows" + runs-on: windows-2019 + needs: [build-lib_Windows, swig_Windows] + strategy: + matrix: + build-flavor: ["tf", "tflite"] + steps: + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + path-type: inherit + update: true + install: >- + make + tar + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/download-artifact@v2 + with: + name: "native_client.${{ matrix.build-flavor }}.tar.xz" + path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ + - run: | + pushd tensorflow/bazel-bin/native_client/ + "C:/Program Files/7-Zip/7z.exe" x native_client.tar.xz -so | "C:/Program Files/7-Zip/7z.exe" x -aoa -si -ttar -o`pwd` + ls -hal + popd + - uses: actions/download-artifact@v2 + with: + name: "swig_Windows" + path: ${{ github.workspace }}/native_client/ds-swig/ + - name: Link ds-swig into swig + run: | + set -ex + ls -hal native_client/ds-swig/bin + ln -s ds-swig.exe native_client/ds-swig/bin/swig.exe + chmod +x native_client/ds-swig/bin/ds-swig.exe native_client/ds-swig/bin/swig.exe + - uses: actions/setup-node@v2 + with: + node-version: 12 + - uses: ./.github/actions/node-build + with: + nodejs_versions: "10.0.0 11.0.0 12.7.0 13.0.0 14.0.0 15.0.0" + electronjs_versions: "5.0.13 6.0.12 6.1.7 7.0.1 7.1.8 8.0.1 9.0.1 9.1.0 9.2.0 10.0.0 10.1.0 11.0.0 12.0.0" + - uses: actions/upload-artifact@v2 + with: + name: "nodewrapper-${{ matrix.build-flavor }}.tar.gz" + path: ${{ github.workspace }}/native_client/javascript/wrapper.tar.gz + - uses: actions/upload-artifact@v2 + with: + name: "deepspeech-${{ matrix.build-flavor }}.tgz" + path: ${{ github.workspace }}/native_client/javascript/deepspeech-*.tgz + test-cpp-Windows: + name: "Test C++ binary on Windows" + runs-on: windows-2019 + needs: [build-lib_Windows, train-test-model] + strategy: + matrix: + build-flavor: ["tf", "tflite"] + env: + TASKCLUSTER_TMP_DIR: tmp/ + DEEPSPEECH_TEST_MODEL: tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Download native_client.tar.xz + uses: actions/download-artifact@v2 + with: + name: "native_client.${{ matrix.build-flavor }}.tar.xz" + path: ${{ env.TASKCLUSTER_TMP_DIR }} + - name: Extract native_client.tar.xz + run: | + mkdir -p ${{ env.TASKCLUSTER_TMP_DIR }}/ds + pushd ${{ env.TASKCLUSTER_TMP_DIR }}/ds + "C:/Program Files/7-Zip/7z.exe" x ../native_client.tar.xz -so | "C:/Program Files/7-Zip/7z.exe" x -aoa -si -ttar -o`pwd` + popd + - name: Download trained test model + uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-16k.zip" + path: ${{ env.TASKCLUSTER_TMP_DIR }} + - run: | + ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + - uses: ./.github/actions/run-tests + with: + runtime: "cppwin" + build-flavor: ${{ matrix.build-flavor }} + bitrate: "16k" + model-kind: "" + test-py-Windows: + name: "Test Python bindings on Windows" + runs-on: windows-2019 + needs: [ build-python-Windows, train-test-model ] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + python-version: [3.6.8, 3.7.9, 3.8.8, 3.9.2] + build-flavor: ["tf", "tflite"] + models: ["test", "prod"] + bitrate: ["8k", "16k"] + env: + TASKCLUSTER_TMP_DIR: tmp/ + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm + DEEPSPEECH_TEST_MODEL: tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + path-type: inherit + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: ./.github/actions/win-install-sox + - uses: actions/download-artifact@v2 + with: + name: "deepspeech-${{ matrix.build-flavor }}-${{ matrix.python-version }}.whl" + path: ${{ env.TASKCLUSTER_TMP_DIR }} + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.TASKCLUSTER_TMP_DIR }} + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + python -m pip install --only-binary :all: --upgrade ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.whl + - uses: ./.github/actions/run-tests + with: + runtime: "python" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + test-nodejs-Windows: + name: "Test NodeJS bindings on Windows" + runs-on: windows-2019 + needs: [ build-nodejs-Windows, train-test-model ] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + nodejs-version: [10, 12, 14, 15] + build-flavor: ["tf", "tflite"] + models: ["test"] + bitrate: ["16k"] + env: + TASKCLUSTER_TMP_DIR: tmp/ + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm + DEEPSPEECH_TEST_MODEL: tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + path-type: inherit + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.nodejs-version }} + - uses: ./.github/actions/win-install-sox + - uses: actions/download-artifact@v2 + with: + name: "deepspeech-${{ matrix.build-flavor }}.tgz" + path: ${{ env.TASKCLUSTER_TMP_DIR }} + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.TASKCLUSTER_TMP_DIR }} + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz + - uses: ./.github/actions/run-tests + with: + runtime: "node" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + test-electronjs-Windows: + name: "Test ElectronJS bindings on Windows" + runs-on: windows-2019 + needs: [ build-nodejs-Windows, train-test-model ] + if: ${{ github.event_name == 'pull_request' }} + strategy: + matrix: + electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0] + build-flavor: ["tf", "tflite"] + models: ["test"] + bitrate: ["16k"] + env: + TASKCLUSTER_TMP_DIR: tmp/ + DEEPSPEECH_PROD_MODEL: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pb + DEEPSPEECH_PROD_MODEL_MMAP: https://github.com/reuben/DeepSpeech/releases/download/v0.7.0-alpha.3/output_graph.pbmm + DEEPSPEECH_TEST_MODEL: tmp/output_graph.pb + EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988" + steps: + - name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front + run: echo "D:\a\_temp\msys\msys64\usr\bin" >> $GITHUB_PATH + - uses: msys2/setup-msys2@v2 + with: + msystem: MSYS + path-type: inherit + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-node@v2 + with: + node-version: 12 + - uses: ./.github/actions/win-install-sox + - uses: actions/download-artifact@v2 + with: + name: "deepspeech-${{ matrix.build-flavor }}.tgz" + path: ${{ env.TASKCLUSTER_TMP_DIR }} + - uses: actions/download-artifact@v2 + with: + name: "test-model.${{ matrix.build-flavor }}-${{ matrix.bitrate }}.zip" + path: ${{ env.TASKCLUSTER_TMP_DIR }} + if: matrix.models == 'test' + - run: | + ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + if: matrix.models == 'test' + - name: Install deepspeech NodeJS package + run: | + ls -hal ${{ env.TASKCLUSTER_TMP_DIR }}/ + npm install ${{ env.TASKCLUSTER_TMP_DIR }}/deepspeech*.tgz + - run: | + npm install electron@${{ matrix.electronjs-version }} + - uses: ./.github/actions/run-tests + with: + runtime: "electronjs" + build-flavor: ${{ matrix.build-flavor }} + bitrate: ${{ matrix.bitrate }} + model-kind: ${{ matrix.models }} + timeout-minutes: 5 diff --git a/ci_scripts/all-utils.sh b/ci_scripts/all-utils.sh index ffd3d191..6dbccb77 100755 --- a/ci_scripts/all-utils.sh +++ b/ci_scripts/all-utils.sh @@ -109,7 +109,14 @@ verify_bazel_rebuild() symlink_electron() { - ln -s Electron.app/Contents/MacOS/Electron node_modules/electron/dist/node + if [ "${OS}" = "Darwin" ]; then + ln -s Electron.app/Contents/MacOS/Electron node_modules/electron/dist/node + else + ln -s electron "node_modules/electron/dist/node" + if [ -f "node_modules/electron/dist/chrome-sandbox" ]; then + export ELECTRON_DISABLE_SANDBOX=1 + fi + fi } export_node_bin_path() diff --git a/ci_scripts/all-vars.sh b/ci_scripts/all-vars.sh index 49cc71ed..906232de 100755 --- a/ci_scripts/all-vars.sh +++ b/ci_scripts/all-vars.sh @@ -10,6 +10,7 @@ if [ "${OS}" = "Linux" ]; then fi; if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + export TASKCLUSTER_TASK_DIR="$(cygpath ${TASKCLUSTER_TASK_DIR})" export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} export PYENV_ROOT="${TASKCLUSTER_TASK_DIR}/pyenv-root" export PLATFORM_EXE_SUFFIX=.exe diff --git a/ci_scripts/cppwin_tflite-tests.sh b/ci_scripts/cppwin_tflite-tests.sh new file mode 100755 index 00000000..884ced1d --- /dev/null +++ b/ci_scripts/cppwin_tflite-tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/all-vars.sh +source $(dirname "$0")/all-utils.sh +source $(dirname "$0")/asserts.sh + +bitrate=$1 +set_ldc_sample_filename "${bitrate}" + +model_source=${DEEPSPEECH_TEST_MODEL//.pb/.tflite} +model_name=$(basename "${model_source}") +model_name_mmap=$(basename "${model_source}") +export DATA_TMP_DIR=${TASKCLUSTER_TMP_DIR} + +download_material "${TASKCLUSTER_TMP_DIR}/ds" + +export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH + +check_versions + +run_basic_inference_tests diff --git a/ci_scripts/tf-build.sh b/ci_scripts/tf-build.sh index f5477467..0f21f03e 100755 --- a/ci_scripts/tf-build.sh +++ b/ci_scripts/tf-build.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex +set -o pipefail source $(dirname $0)/tf-vars.sh @@ -21,7 +22,10 @@ pushd ${DS_ROOT_TASK}/tensorflow/ fi; case "$1" in - "--linux-cpu"|"--darwin-cpu"|"--windows-cpu") + "--windows-cpu") + echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIBDEEPSPEECH} ${BUILD_TARGET_LITE_LIB} --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" + ;; + "--linux-cpu"|"--darwin-cpu") echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB} ;; "--linux-cuda"|"--windows-cuda") diff --git a/ci_scripts/tf-setup.sh b/ci_scripts/tf-setup.sh index 80c6312f..b124924d 100755 --- a/ci_scripts/tf-setup.sh +++ b/ci_scripts/tf-setup.sh @@ -109,4 +109,4 @@ mkdir -p ${TASKCLUSTER_ARTIFACTS} || true # Taken from https://www.tensorflow.org/install/source # Only future is needed for our builds, as we don't build the Python package -pip install -U --user future==0.17.1 || true +python -m pip install -U --user future==0.17.1 || true diff --git a/ci_scripts/tf-vars.sh b/ci_scripts/tf-vars.sh index 9e08f00a..f29c5ec3 100755 --- a/ci_scripts/tf-vars.sh +++ b/ci_scripts/tf-vars.sh @@ -40,9 +40,6 @@ elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then export TASKCLUSTER_ARTIFACTS="$(cygpath ${TASKCLUSTER_ARTIFACTS})" export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} - export BAZEL_VC='C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC' - export BAZEL_SH='C:\builds\tc-workdir\msys64\usr\bin\bash' - export TC_WIN_BUILD_PATH='C:\builds\tc-workdir\msys64\usr\bin;C:\Python36' export MSYS2_ARG_CONV_EXCL='//' mkdir -p ${TASKCLUSTER_TASK_DIR}/tmp/ @@ -119,19 +116,15 @@ export TF_NEED_ROCM=0 # This should be gcc-5, hopefully. CUDA and TensorFlow might not be happy, otherwise. export GCC_HOST_COMPILER_PATH=/usr/bin/gcc -if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then - export PYTHON_BIN_PATH=C:/Python36/python.exe -else - if [ "${OS}" = "Linux" ]; then - source /etc/os-release - if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then - export PYTHON_BIN_PATH=/usr/bin/python3 - else - export PYTHON_BIN_PATH=/usr/bin/python2.7 - fi +if [ "${OS}" = "Linux" ]; then + source /etc/os-release + if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then + export PYTHON_BIN_PATH=/usr/bin/python3 else - export PYTHON_BIN_PATH=/usr/bin/python2.7 + export PYTHON_BIN_PATH=/usr/bin/python2.7 fi +elif [ "${OS}" != "${TC_MSYS_VERSION}" ]; then + export PYTHON_BIN_PATH=/usr/bin/python2.7 fi ## Below, define or export some build variables @@ -186,15 +179,7 @@ fi BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV" -if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then - # Somehow, even with Python being in the PATH, Bazel on windows struggles - # with '/usr/bin/env python' ... - # - # We also force TMP/TEMP otherwise Bazel will pick default Windows one - # under %USERPROFILE%\AppData\Local\Temp and with 8.3 file format convention - # it messes with cxx_builtin_include_directory - BAZEL_EXTRA_FLAGS="--action_env=PATH=${TC_WIN_BUILD_PATH} --action_env=TEMP=${TEMP} --action_env=TMP=${TMP}" -else +if [ "${OS}" != "${TC_MSYS_VERSION}" ]; then BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl --copt=-fvisibility=hidden" fi @@ -211,3 +196,4 @@ BUILD_TARGET_GRAPH_BENCHMARK="//tensorflow/tools/benchmark:benchmark_model" BUILD_TARGET_TOCO="//tensorflow/lite/toco:toco" BUILD_TARGET_LITE_BENCHMARK="//tensorflow/lite/tools/benchmark:benchmark_model" BUILD_TARGET_LITE_LIB="//tensorflow/lite/c:libtensorflowlite_c.so" +BUILD_TARGET_LIBDEEPSPEECH="//native_client:libdeepspeech.so" diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index 2a565b69..dcaaae7f 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -45,16 +45,16 @@ workspace_status.cc: # Enforce PATH here because swig calls from build_ext looses track of some # variables over several runs bindings: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP) - pip3 install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0 - DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python3 ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + python -m pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0 + DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) find temp_build -type f -name "*.o" -delete - DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python3 ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build bindings-debug: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP) - pip3 install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0 - DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python3 ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + python -m pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==45.0.0 + DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) $(GENERATE_DEBUG_SYMS) find temp_build -type f -name "*.o" -delete - DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python3 ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 168e36ed..02c883bd 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -55,7 +55,7 @@ PYTHON_PLATFORM_NAME ?= --plat-name manylinux1_x86_64 endif endif -ifeq ($(TARGET),host-win) +ifeq ($(findstring _NT,$(OS)),_NT) TOOLCHAIN := '$(VCToolsInstallDir)\bin\Hostx64\x64\' TOOL_CC := cl.exe TOOL_CXX := cl.exe