From 8f0b759103eee49162cefa3c92033f271aaf6c62 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 29 Jul 2021 18:58:52 +0200 Subject: [PATCH 1/3] Build TensorFlow+libstt+Py pkgs on manylinux_2_24 --- .github/actions/python-build/action.yml | 18 +--- .github/workflows/build-and-test.yml | 134 ++++++++++++++++++++---- ci_scripts/build-utils.sh | 1 - ci_scripts/package-utils.sh | 13 +++ ci_scripts/tf-build.sh | 9 -- ci_scripts/tf-setup.sh | 27 ++--- ci_scripts/tf-vars.sh | 17 +-- native_client/definitions.mk | 9 +- 8 files changed, 152 insertions(+), 76 deletions(-) diff --git a/.github/actions/python-build/action.yml b/.github/actions/python-build/action.yml index a30df7b8..fad19cd9 100644 --- a/.github/actions/python-build/action.yml +++ b/.github/actions/python-build/action.yml @@ -33,11 +33,6 @@ runs: - run: | python3 --version pip3 --version - python3 -m pip install virtualenv - python3 -m virtualenv stt-build - shell: bash - - run: | - mkdir -p wheels shell: bash - run: | set -xe @@ -45,7 +40,8 @@ runs: PROJECT_NAME="stt" OS=$(uname) - if [ "${OS}" = "Linux" ]; then + if [ "${OS}" = "Linux" -a "${{ inputs.target }}" != "host" ]; then + python3 -m venv stt-build source stt-build/bin/activate fi @@ -59,14 +55,4 @@ runs: RASPBIAN=${{ inputs.chroot }} \ SETUP_FLAGS="--project_name ${PROJECT_NAME}" \ bindings-clean bindings - - if [ "${OS}" = "Linux" ]; then - deactivate - fi - shell: bash - - run: | - cp native_client/python/dist/*.whl wheels - shell: bash - - run: | - make -C native_client/python/ bindings-clean shell: bash diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index ab3ab4ac..990a5249 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -111,6 +111,48 @@ jobs: with: name: ${{ github.job }} path: ${{ github.workspace }}/build-static/ + libsox3_Linux: + name: "Lin|Build libsox3" + runs-on: ubuntu-20.04 + container: + image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c + volumes: + - ${{ github.workspace }}:${{ github.workspace }} + steps: + - run: | + curl -sSL https://github.com/coqui-ai/STT/releases/download/v0.10.0-alpha.7/sox-14.4.2.tar.bz2 | tar xjf - + - run: | + mkdir -p sox-build/ + - uses: actions/cache@v2 + id: sox-build-cache + with: + path: sox-build/ + key: sox-1-${{ runner.os }}-${{ env.sox_hash }} + - name: Configure + run: | + cd sox-14.4.2 + ./configure \ + --with-dyn-default --enable-dl-sndfile --enable-dl-amrnb \ + --without-magic --without-png --without-ladspa --without-mad \ + --without-id3tag --without-lame --without-twolame \ + --without-libltdl --disable-openmp --disable-dependency-tracking \ + --with-gsm=no --with-lpc10=no --with-oss=no \ + --prefix=${{ github.workspace }}/sox-build/ + if: steps.sox-build-cache.outputs.cache-hit != 'true' + - name: Make + run: | + cd sox-14.4.2 + make -j + if: steps.sox-build-cache.outputs.cache-hit != 'true' + - name: Make install + run: | + cd sox-14.4.2 + make install + if: steps.sox-build-cache.outputs.cache-hit != 'true' + - uses: actions/upload-artifact@v2 + with: + name: ${{ github.job }} + path: ${{ github.workspace }}/sox-build/ build-ctc-decoder-Linux: name: "Lin|Build CTC decoder Python package" needs: [ swig_Linux ] @@ -240,7 +282,7 @@ jobs: - id: get_cache_key uses: ./.github/actions/get_cache_key with: - extras: "3" + extras: "4" - id: check_artifact_exists uses: ./.github/actions/check_artifact_exists with: @@ -249,6 +291,10 @@ jobs: name: "Lin|Build TensorFlow (opt)" needs: tensorflow_opt-Linux runs-on: ubuntu-20.04 + container: + image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c + volumes: + - ${{ github.workspace }}:${{ github.workspace }} steps: - run: true if: needs.tensorflow_opt-Linux.outputs.status == 'found' @@ -257,11 +303,17 @@ jobs: fetch-depth: 0 submodules: 'recursive' if: needs.tensorflow_opt-Linux.outputs.status == 'missing' - - run: | - sudo apt-get install -y --no-install-recommends pixz + - name: Install dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends xz-utils if: needs.tensorflow_opt-Linux.outputs.status == 'missing' - uses: ./.github/actions/setup-tensorflow if: needs.tensorflow_opt-Linux.outputs.status == 'missing' + - name: Setup venv + run: | + /opt/python/cp37-cp37m/bin/python -m venv /tmp/venv + echo "/tmp/venv/bin" >> $GITHUB_PATH - uses: ./.github/actions/build-tensorflow with: flavor: "--linux-cpu" @@ -276,7 +328,11 @@ jobs: build-lib_Linux: name: "Lin|Build libstt+client" runs-on: ubuntu-20.04 - needs: [ build-tensorflow-Linux, tensorflow_opt-Linux ] + needs: [build-tensorflow-Linux, tensorflow_opt-Linux, libsox3_Linux] + container: + image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c + volumes: + - ${{ github.workspace }}:${{ github.workspace }} steps: - uses: actions/checkout@v2 with: @@ -286,13 +342,21 @@ jobs: name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz path: ${{ github.workspace }}/ download: true + - name: Install dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends xz-utils zip - run: | tar --skip-old-files -xf ${{ needs.tensorflow_opt-linux.outputs.cache_key }}.tar.xz rm ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz - - run: | - sudo apt-get install -y --no-install-recommends make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev - - run: | - git status + - name: Setup venv + run: | + /opt/python/cp37-cp37m/bin/python -m venv /tmp/venv + echo "/tmp/venv/bin" >> $GITHUB_PATH + - uses: actions/download-artifact@v2 + with: + name: "libsox3_Linux" + path: ${{ github.workspace }}/sox-build/ - uses: ./.github/actions/host-build - uses: ./.github/actions/package - uses: actions/upload-artifact@v2 @@ -306,10 +370,14 @@ jobs: build-python-Linux: name: "Lin|Build Python bindings" runs-on: ubuntu-20.04 - needs: [ build-lib_Linux, swig_Linux ] + needs: [build-lib_Linux, swig_Linux] strategy: matrix: python-version: [3.6, 3.7, 3.8, 3.9] + container: + image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c + volumes: + - ${{ github.workspace }}:${{ github.workspace }} steps: - uses: actions/checkout@v2 with: @@ -318,7 +386,12 @@ jobs: with: name: "native_client.tflite.Linux.tar.xz" path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ - - run: | + - name: Install dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends xz-utils + - name: Extract native_client.tar.xz + run: | cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/ tar xf native_client.tar.xz mkdir -p ../tensorflow/lite @@ -334,9 +407,10 @@ jobs: ls -hal ${{ github.workspace }}/native_client/ds-swig/bin ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - name: Setup venv + run: | + /opt/_internal/cpython-${{ matrix.python-version }}*/bin/python -m venv /tmp/venv-${{ matrix.python-version }} + echo "/tmp/venv-${{ matrix.python-version }}/bin" >> $GITHUB_PATH - id: get_numpy uses: ./.github/actions/numpy_vers with: @@ -345,10 +419,13 @@ jobs: with: numpy_build: "${{ steps.get_numpy.outputs.build_version }}" numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}" + - name: Auditwheel repair + run: | + auditwheel repair native_client/python/dist/*.whl - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-Linux.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/wheelhouse/*.whl build-nodejs-Linux: name: "Lin|Build NodeJS and ElectronJS" runs-on: ubuntu-20.04 @@ -497,6 +574,7 @@ jobs: nodejs-version: [10, 12, 14, 16] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -550,6 +628,7 @@ jobs: electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -1083,7 +1162,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-macOS.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-macOS: name: "Mac|Build NodeJS and ElectronJS" runs-on: macos-10.15 @@ -1230,6 +1309,7 @@ jobs: nodejs-version: [10, 12, 14, 16] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -1541,7 +1621,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-Windows.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-Windows: name: "Win|Build NodeJS/ElectronJS" runs-on: windows-2019 @@ -1717,6 +1797,7 @@ jobs: nodejs-version: [10, 12, 14, 16] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -1892,6 +1973,7 @@ jobs: nodejs-version: [10, 16] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -1945,6 +2027,7 @@ jobs: electronjs-version: [5.0.13, 12.0.0] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2000,6 +2083,7 @@ jobs: nodejs-version: [10, 16] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2051,6 +2135,7 @@ jobs: electronjs-version: [5.0.13, 12.0.0] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2104,6 +2189,7 @@ jobs: nodejs-version: [10, 16] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2167,6 +2253,7 @@ jobs: electronjs-version: [5.0.13, 12.0.0] models: ["test", "prod"] bitrate: ["8k", "16k"] + fail-fast: false env: CI_TMP_DIR: tmp/ STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb @@ -2391,7 +2478,7 @@ jobs: name: "libstt.tflite.linux.aarch64.zip" path: ${{ github.workspace }}/artifacts/libstt.zip build-python-LinuxArmv7: - name: "LinArmv7|Build python bindings" + name: "LinArmv7|Build Python bindings" runs-on: ubuntu-20.04 needs: [ build-lib_LinuxArmv7, swig_Linux, tensorflow_opt-LinuxArmv7 ] strategy: @@ -2447,6 +2534,9 @@ jobs: uses: ./.github/actions/numpy_vers with: pyver: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install -U pip setuptools wheel - uses: ./.github/actions/python-build with: numpy_build: "${{ steps.get_numpy.outputs.build_version }}" @@ -2456,7 +2546,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-armv7.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-LinuxArmv7: name: "LinArmv7|Build NodeJS and ElectronJS" runs-on: ubuntu-20.04 @@ -2531,7 +2621,7 @@ jobs: name: "stt_intermediate-tflite-armv7.tgz" path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz build-python-LinuxAarch64: - name: "LinAarch64|Build python bindings" + name: "LinAarch64|Build Python bindings" runs-on: ubuntu-20.04 needs: [ build-lib_LinuxAarch64, swig_Linux, tensorflow_opt-LinuxAarch64 ] strategy: @@ -2589,6 +2679,9 @@ jobs: uses: ./.github/actions/numpy_vers with: pyver: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install -U pip setuptools wheel - uses: ./.github/actions/python-build with: numpy_build: "${{ steps.get_numpy.outputs.build_version }}" @@ -2598,7 +2691,7 @@ jobs: - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-aarch64.whl" - path: ${{ github.workspace }}/wheels/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-LinuxAarch64: name: "LinAarch64|Build NodeJS and ElectronJS" runs-on: ubuntu-20.04 @@ -2834,6 +2927,7 @@ jobs: nodejs-version: [10, 12, 14, 16] models: ["test"] bitrate: ["16k"] + fail-fast: false env: CI_TMP_DIR: ${{ github.workspace }}/tmp DEBIAN_FRONTEND: "noninteractive" diff --git a/ci_scripts/build-utils.sh b/ci_scripts/build-utils.sh index df57e8bc..77106b3a 100755 --- a/ci_scripts/build-utils.sh +++ b/ci_scripts/build-utils.sh @@ -11,7 +11,6 @@ do_bazel_build() bazel ${BAZEL_OUTPUT_USER_ROOT} build \ -s --explain bazel_explain.log --verbose_explanations \ - --experimental_strict_action_env \ --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" \ -c ${_opt_or_dbg} ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS} diff --git a/ci_scripts/package-utils.sh b/ci_scripts/package-utils.sh index c59e6f8c..088b28d1 100755 --- a/ci_scripts/package-utils.sh +++ b/ci_scripts/package-utils.sh @@ -34,12 +34,18 @@ package_native_client() win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so.if.lib" fi; + libsox_lib="" + if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then + libsox_lib="-C ${stt_dir}/sox-build/lib libsox.so.3" + fi + ${TAR} --verbose -cf - \ --transform='flags=r;s|README.coqui|KenLM_License_Info.txt|' \ -C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so \ -C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so \ -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so \ ${win_lib} \ + ${libsox_lib} \ -C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \ -C ${stt_dir}/ LICENSE \ -C ${stt_dir}/native_client/ stt${PLATFORM_EXE_SUFFIX} \ @@ -85,6 +91,7 @@ package_native_client_ndk() package_libstt_as_zip() { tensorflow_dir=${DS_TFDIR} + stt_dir=${DS_DSDIR} artifacts_dir=${CI_ARTIFACTS_DIR} artifact_name=$1 @@ -99,8 +106,14 @@ package_libstt_as_zip() echo "Please specify artifact name." fi; + libsox_lib="" + if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then + libsox_lib="${stt_dir}/sox-build/lib/libsox.so.3" + fi + ${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" \ ${tensorflow_dir}/bazel-bin/native_client/libstt.so \ ${tensorflow_dir}/bazel-bin/native_client/libkenlm.so \ + ${libsox_lib} \ ${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so } diff --git a/ci_scripts/tf-build.sh b/ci_scripts/tf-build.sh index 0f5aba63..1d1fefbe 100755 --- a/ci_scripts/tf-build.sh +++ b/ci_scripts/tf-build.sh @@ -8,13 +8,6 @@ source $(dirname $0)/tf-vars.sh pushd ${DS_ROOT_TASK}/tensorflow/ BAZEL_BUILD="bazel ${BAZEL_OUTPUT_USER_ROOT} build -s" - # Start a bazel process to ensure reliability on Windows and avoid: - # FATAL: corrupt installation: file 'c:\builds\tc-workdir\.bazel_cache/install/6b1660721930e9d5f231f7d2a626209b/_embedded_binaries/build-runfiles.exe' missing. - bazel ${BAZEL_OUTPUT_USER_ROOT} info - - # Force toolchain sync (useful on macOS ?) - bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure - MAYBE_DEBUG=$2 OPT_OR_DBG="-c opt" if [ "${MAYBE_DEBUG}" = "dbg" ]; then @@ -47,6 +40,4 @@ pushd ${DS_ROOT_TASK}/tensorflow/ echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB} ;; esac - - bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown popd diff --git a/ci_scripts/tf-setup.sh b/ci_scripts/tf-setup.sh index cde2ad76..276e66d1 100755 --- a/ci_scripts/tf-setup.sh +++ b/ci_scripts/tf-setup.sh @@ -17,7 +17,7 @@ download() { fname=`basename $1` - ${WGET} $1 -O ${DS_ROOT_TASK}/dls/$fname && echo "$2 ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} - + ${CURL} -sSL -o ${DS_ROOT_TASK}/dls/$fname $1 && echo "$2 ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} - } # Download stuff @@ -34,19 +34,15 @@ ls -hal ${DS_ROOT_TASK}/dls/ # Install Bazel in ${DS_ROOT_TASK}/bin BAZEL_INSTALL_FILENAME=$(basename "${BAZEL_URL}") -if [ "${OS}" = "Linux" ]; then - BAZEL_INSTALL_FLAGS="--user" -elif [ "${OS}" = "Darwin" ]; then - BAZEL_INSTALL_FLAGS="--bin=${DS_ROOT_TASK}/bin --base=${DS_ROOT_TASK}/.bazel" -fi; mkdir -p ${DS_ROOT_TASK}/bin || true -pushd ${DS_ROOT_TASK}/bin - if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then - cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel.exe - else - /bin/bash ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${BAZEL_INSTALL_FLAGS} - fi -popd + +SUFFIX="" +if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then + SUFFIX=".exe" +fi + +cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel${SUFFIX} +chmod +x ${DS_ROOT_TASK}/bin/bazel${SUFFIX} # For debug bazel version @@ -71,8 +67,3 @@ if [ ! -z "${install_android}" ]; then fi mkdir -p ${CI_ARTIFACTS_DIR} || true - - -# Taken from https://www.tensorflow.org/install/source -# Only future is needed for our builds, as we don't build the Python package -python -m pip install -U --user future==0.17.1 || true diff --git a/ci_scripts/tf-vars.sh b/ci_scripts/tf-vars.sh index ebf4a5fd..191941bc 100755 --- a/ci_scripts/tf-vars.sh +++ b/ci_scripts/tf-vars.sh @@ -6,8 +6,8 @@ export OS=$(uname) if [ "${OS}" = "Linux" ]; then export DS_ROOT_TASK=${CI_TASK_DIR} - BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-linux-x86_64.sh - BAZEL_SHA256=7ba815cbac712d061fe728fef958651512ff394b2708e89f79586ec93d1185ed + BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-linux-amd64 + BAZEL_SHA256=4cb534c52cdd47a6223d4596d530e7c9c785438ab3b0a49ff347e991c210b2cd ANDROID_NDK_URL=https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip ANDROID_NDK_SHA256=4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd @@ -38,8 +38,8 @@ elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then export TEMP=${CI_TASK_DIR}/tmp/ export TMP=${CI_TASK_DIR}/tmp/ - BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-windows-x86_64.exe - BAZEL_SHA256=776db1f4986dacc3eda143932f00f7529f9ee65c7c1c004414c44aaa6419d0e9 + BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-windows-amd64.exe + BAZEL_SHA256=9a89e6a8cc0a3aea37affcf8c146d8925ffbda1d2290c0c6a845ea81e05de62c TAR=/usr/bin/tar.exe elif [ "${OS}" = "Darwin" ]; then @@ -52,14 +52,15 @@ elif [ "${OS}" = "Darwin" ]; then export DS_ROOT_TASK=${CI_TASK_DIR} - BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-darwin-x86_64.sh - BAZEL_SHA256=5cfa97031b43432b3c742c80e2e01c41c0acdca7ba1052fc8cf1e291271bc9cd + BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-darwin-amd64 + BAZEL_SHA256=e485bbf84532d02a60b0eb23c702610b5408df3a199087a4f2b5e0995bbf2d5a SHA_SUM="shasum -a 256 -c" TAR=gtar fi; WGET=${WGET:-"wget"} +CURL=${CURL:-"curl"} TAR=${TAR:-"tar"} XZ=${XZ:-"xz -9 -T0"} ZIP=${ZIP:-"zip"} @@ -110,8 +111,8 @@ export GCC_HOST_COMPILER_PATH=/usr/bin/gcc if [ "${OS}" = "Linux" ]; then source /etc/os-release - if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then - export PYTHON_BIN_PATH=/usr/bin/python3 + if [ "${ID}" = "debian" -a "${VERSION_ID}" = "9" ]; then + export PYTHON_BIN_PATH=/opt/python/cp37-cp37m/bin/python fi elif [ "${OS}" != "${TC_MSYS_VERSION}" ]; then export PYTHON_BIN_PATH=python diff --git a/native_client/definitions.mk b/native_client/definitions.mk index e630001b..8e8c9d89 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -1,6 +1,7 @@ NC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) TARGET ?= host +ROOT_DIR ?= $(abspath $(NC_DIR)/..) TFDIR ?= $(abspath $(NC_DIR)/../tensorflow) PREFIX ?= /usr/local SO_SEARCH ?= $(TFDIR)/bazel-bin/ @@ -28,7 +29,7 @@ TOOLCHAIN := CFLAGS := CXXFLAGS := LDFLAGS := -SOX_CFLAGS := `pkg-config --cflags sox` +SOX_CFLAGS := -I$(ROOT_DIR)/sox-build/include ifeq ($(OS),Linux) MAGIC_LINK_LZMA := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.so | grep lzma | grep '*UND*' | wc -l) ifneq ($(MAGIC_LINK_LZMA),0) @@ -38,8 +39,7 @@ MAGIC_LINK_BZ2 := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.s ifneq ($(MAGIC_LINK_BZ2),0) MAYBE_LINK_BZ2 := -lbz2 endif # MAGIC_LINK_BZ2 -SOX_CFLAGS += -fopenmp -SOX_LDFLAGS := -Wl,-Bstatic `pkg-config --static --libs sox` -lgsm `pkg-config --static --libs libpng | cut -d' ' -f1` -lz -lmagic $(MAYBE_LINK_LZMA) $(MAYBE_LINK_BZ2) -lltdl -Wl,-Bdynamic -ldl +SOX_LDFLAGS := -L$(ROOT_DIR)/sox-build/lib -lsox else ifeq ($(OS),Darwin) LIBSOX_PATH := $(shell echo `pkg-config --libs-only-L sox | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l sox | sed -e 's/^-l//'`.dylib) LIBOPUSFILE_PATH := $(shell echo `pkg-config --libs-only-L opusfile | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l opusfile | sed -e 's/^-l//'`.dylib) @@ -172,7 +172,7 @@ define copy_missing_libs SRC_FILE=$(1); \ TARGET_LIB_DIR=$(2); \ MANIFEST_IN=$(3); \ - echo "Analyzing $$SRC_FILE copying missing libs to $$SRC_FILE"; \ + echo "Analyzing $$SRC_FILE copying missing libs to $$TARGET_LIB_DIR"; \ echo "Maybe outputting to $$MANIFEST_IN"; \ \ (mkdir $$TARGET_LIB_DIR || true); \ @@ -188,6 +188,7 @@ define copy_missing_libs fi; \ done; \ \ + echo "Missing libs = $$missing_libs"; \ for missing in $$missing_libs; do \ find $(SO_SEARCH) -type f -name "$$missing" -exec cp {} $$TARGET_LIB_DIR \; ; \ chmod +w $$TARGET_LIB_DIR/*.so ; \ From 1a423a4c8d53a088012215d7e175b779603f3766 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Fri, 30 Jul 2021 21:41:36 +0200 Subject: [PATCH 2/3] Force plat name in favor of auditwheel for Python packages Auditwheel can't properly deal with the shared libraries we bundle and ends up copying some of them, ending up with a package with duplicated images. --- .github/workflows/build-and-test.yml | 5 +---- native_client/ctcdecode/Makefile | 8 ++++---- native_client/definitions.mk | 3 +++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 990a5249..2cadbd45 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -419,13 +419,10 @@ jobs: with: numpy_build: "${{ steps.get_numpy.outputs.build_version }}" numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}" - - name: Auditwheel repair - run: | - auditwheel repair native_client/python/dist/*.whl - uses: actions/upload-artifact@v2 with: name: "stt-tflite-${{ matrix.python-version }}-Linux.whl" - path: ${{ github.workspace }}/wheelhouse/*.whl + path: ${{ github.workspace }}/native_client/python/dist/*.whl build-nodejs-Linux: name: "Lin|Build NodeJS and ElectronJS" runs-on: ubuntu-20.04 diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index a069e809..d8899ea7 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -46,15 +46,15 @@ workspace_status.cc: # variables over several runs bindings: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP) python -m pip install --quiet $(PYTHON_PACKAGES) wheel setuptools - DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(SETUP_FLAGS) find temp_build -type f -name "*.o" -delete - DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(SETUP_FLAGS) rm -rf temp_build bindings-debug: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP) python -m pip install --quiet $(PYTHON_PACKAGES) wheel setuptools - DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(SETUP_FLAGS) $(GENERATE_DEBUG_SYMS) find temp_build -type f -name "*.o" -delete - DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(SETUP_FLAGS) rm -rf temp_build diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 8e8c9d89..1f55b83b 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -50,6 +50,9 @@ else SOX_LDFLAGS := `pkg-config --libs sox` endif # OS others PYTHON_PACKAGES := numpy${NUMPY_BUILD_VERSION} +ifeq ($(OS),Linux) +PYTHON_PLATFORM_NAME ?= --plat-name manylinux_2_24_x86_64 +endif endif ifeq ($(findstring _NT,$(OS)),_NT) From 9d44e2f5066bf6b1863174c588a7bda938ea6ab0 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Sun, 1 Aug 2021 23:25:18 +0200 Subject: [PATCH 3/3] Disable wrapping of struct ctors to workaround NodeJS 16.6 ABI break --- native_client/javascript/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/native_client/javascript/Makefile b/native_client/javascript/Makefile index c4d1dd82..dcfecb9e 100644 --- a/native_client/javascript/Makefile +++ b/native_client/javascript/Makefile @@ -63,3 +63,6 @@ npm-pack: clean package.json index.ts npm-dev stt_wrap.cxx: stt.i ds-swig $(DS_SWIG_ENV) swig -c++ -javascript -node stt.i + # Hack: disable wrapping of constructors to avoid NodeJS 16.6 ABI compat break + sed -i.bak '/SetCallHandler/d' stt_wrap.cxx + rm stt_wrap.cxx.bak