Build TensorFlow+libstt+Py pkgs on manylinux_2_24

.github/actions/python-build/action.yml

@@ -33,11 +33,6 @@ runs:
     - run: |
         python3 --version
         pip3 --version
-        python3 -m pip install virtualenv
-        python3 -m virtualenv stt-build
-      shell: bash
-    - run: |
-        mkdir -p wheels
       shell: bash
     - run: |
         set -xe
@@ -45,7 +40,8 @@ runs:
         PROJECT_NAME="stt"
 
         OS=$(uname)
-        if [ "${OS}" = "Linux" ]; then
+        if [ "${OS}" = "Linux" -a "${{ inputs.target }}" != "host" ]; then
+          python3 -m venv stt-build
           source stt-build/bin/activate
         fi
 
@@ -59,14 +55,4 @@ runs:
             RASPBIAN=${{ inputs.chroot }} \
             SETUP_FLAGS="--project_name ${PROJECT_NAME}" \
             bindings-clean bindings
-
-        if [ "${OS}" = "Linux" ]; then
-          deactivate
-        fi
-      shell: bash
-    - run: |
-        cp native_client/python/dist/*.whl wheels
-      shell: bash
-    - run: |
-        make -C native_client/python/ bindings-clean
       shell: bash

.github/workflows/build-and-test.yml

@@ -111,6 +111,48 @@ jobs:
         with:
           name: ${{ github.job }}
           path: ${{ github.workspace }}/build-static/
+  libsox3_Linux:
+    name: "Lin|Build libsox3"
+    runs-on: ubuntu-20.04
+    container:
+      image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
+      volumes:
+        - ${{ github.workspace }}:${{ github.workspace }}
+    steps:
+      - run: |
+          curl -sSL https://github.com/coqui-ai/STT/releases/download/v0.10.0-alpha.7/sox-14.4.2.tar.bz2 | tar xjf -
+      - run: |
+          mkdir -p sox-build/
+      - uses: actions/cache@v2
+        id: sox-build-cache
+        with:
+          path: sox-build/
+          key: sox-1-${{ runner.os }}-${{ env.sox_hash }}
+      - name: Configure
+        run: |
+          cd sox-14.4.2
+          ./configure \
+            --with-dyn-default --enable-dl-sndfile --enable-dl-amrnb \
+            --without-magic --without-png --without-ladspa --without-mad \
+            --without-id3tag --without-lame --without-twolame \
+            --without-libltdl --disable-openmp --disable-dependency-tracking \
+            --with-gsm=no --with-lpc10=no --with-oss=no \
+            --prefix=${{ github.workspace }}/sox-build/
+        if: steps.sox-build-cache.outputs.cache-hit != 'true'
+      - name: Make
+        run: |
+          cd sox-14.4.2
+          make -j
+        if: steps.sox-build-cache.outputs.cache-hit != 'true'
+      - name: Make install
+        run: |
+          cd sox-14.4.2
+          make install
+        if: steps.sox-build-cache.outputs.cache-hit != 'true'
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ github.job }}
+          path: ${{ github.workspace }}/sox-build/
   build-ctc-decoder-Linux:
     name: "Lin|Build CTC decoder Python package"
     needs: [ swig_Linux ]
@@ -240,7 +282,7 @@ jobs:
       - id: get_cache_key
         uses: ./.github/actions/get_cache_key
         with:
-          extras: "3"
+          extras: "4"
       - id: check_artifact_exists
         uses: ./.github/actions/check_artifact_exists
         with:
@@ -249,6 +291,10 @@ jobs:
     name: "Lin|Build TensorFlow (opt)"
     needs: tensorflow_opt-Linux
     runs-on: ubuntu-20.04
+    container:
+      image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
+      volumes:
+        - ${{ github.workspace }}:${{ github.workspace }}
     steps:
       - run: true
         if: needs.tensorflow_opt-Linux.outputs.status == 'found'
@@ -257,11 +303,17 @@ jobs:
           fetch-depth: 0
           submodules: 'recursive'
         if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
-      - run: |
+      - name: Install dependencies
-          sudo apt-get install -y --no-install-recommends pixz
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends xz-utils
         if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
       - uses: ./.github/actions/setup-tensorflow
         if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
+      - name: Setup venv
+        run: |
+          /opt/python/cp37-cp37m/bin/python -m venv /tmp/venv
+          echo "/tmp/venv/bin" >> $GITHUB_PATH
       - uses: ./.github/actions/build-tensorflow
         with:
           flavor: "--linux-cpu"
@@ -276,7 +328,11 @@ jobs:
   build-lib_Linux:
     name: "Lin|Build libstt+client"
     runs-on: ubuntu-20.04
-    needs: [ build-tensorflow-Linux, tensorflow_opt-Linux ]
+    needs: [build-tensorflow-Linux, tensorflow_opt-Linux, libsox3_Linux]
+    container:
+      image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
+      volumes:
+        - ${{ github.workspace }}:${{ github.workspace }}
     steps:
       - uses: actions/checkout@v2
         with:
@@ -286,13 +342,21 @@ jobs:
           name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz
           path: ${{ github.workspace }}/
           download: true
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends xz-utils zip
       - run: |
           tar --skip-old-files -xf ${{ needs.tensorflow_opt-linux.outputs.cache_key }}.tar.xz
           rm ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz
-      - run: |
+      - name: Setup venv
-          sudo apt-get install -y --no-install-recommends make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev
+        run: |
-      - run: |
+          /opt/python/cp37-cp37m/bin/python -m venv /tmp/venv
-          git status
+          echo "/tmp/venv/bin" >> $GITHUB_PATH
+      - uses: actions/download-artifact@v2
+        with:
+          name: "libsox3_Linux"
+          path: ${{ github.workspace }}/sox-build/
       - uses: ./.github/actions/host-build
       - uses: ./.github/actions/package
       - uses: actions/upload-artifact@v2
@@ -306,10 +370,14 @@ jobs:
   build-python-Linux:
     name: "Lin|Build Python bindings"
     runs-on: ubuntu-20.04
-    needs: [ build-lib_Linux, swig_Linux ]
+    needs: [build-lib_Linux, swig_Linux]
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8, 3.9]
+    container:
+      image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
+      volumes:
+        - ${{ github.workspace }}:${{ github.workspace }}
     steps:
       - uses: actions/checkout@v2
         with:
@@ -318,7 +386,12 @@ jobs:
         with:
           name: "native_client.tflite.Linux.tar.xz"
           path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
-      - run: |
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends xz-utils
+      - name: Extract native_client.tar.xz
+        run: |
           cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
           tar xf native_client.tar.xz
           mkdir -p ../tensorflow/lite
@@ -334,9 +407,10 @@ jobs:
           ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
           ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
           chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
-      - uses: actions/setup-python@v2
+      - name: Setup venv
-        with:
+        run: |
-          python-version: ${{ matrix.python-version }}
+          /opt/_internal/cpython-${{ matrix.python-version }}*/bin/python -m venv /tmp/venv-${{ matrix.python-version }}
+          echo "/tmp/venv-${{ matrix.python-version }}/bin" >> $GITHUB_PATH
       - id: get_numpy
         uses: ./.github/actions/numpy_vers
         with:
@@ -345,10 +419,13 @@ jobs:
         with:
           numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
           numpy_dep: "${{ steps.get_numpy.outputs.dep_version }}"
+      - name: Auditwheel repair
+        run: |
+          auditwheel repair native_client/python/dist/*.whl
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-Linux.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/wheelhouse/*.whl
   build-nodejs-Linux:
     name: "Lin|Build NodeJS and ElectronJS"
     runs-on: ubuntu-20.04
@@ -497,6 +574,7 @@ jobs:
         nodejs-version: [10, 12, 14, 16]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -550,6 +628,7 @@ jobs:
         electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -1083,7 +1162,7 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-macOS.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/native_client/python/dist/*.whl
   build-nodejs-macOS:
     name: "Mac|Build NodeJS and ElectronJS"
     runs-on: macos-10.15
@@ -1230,6 +1309,7 @@ jobs:
         nodejs-version: [10, 12, 14, 16]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -1541,7 +1621,7 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-Windows.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/native_client/python/dist/*.whl
   build-nodejs-Windows:
     name: "Win|Build NodeJS/ElectronJS"
     runs-on: windows-2019
@@ -1717,6 +1797,7 @@ jobs:
         nodejs-version: [10, 12, 14, 16]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -1892,6 +1973,7 @@ jobs:
         nodejs-version: [10, 16]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -1945,6 +2027,7 @@ jobs:
         electronjs-version: [5.0.13, 12.0.0]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2000,6 +2083,7 @@ jobs:
         nodejs-version: [10, 16]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2051,6 +2135,7 @@ jobs:
         electronjs-version: [5.0.13, 12.0.0]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2104,6 +2189,7 @@ jobs:
         nodejs-version: [10, 16]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2167,6 +2253,7 @@ jobs:
         electronjs-version: [5.0.13, 12.0.0]
         models: ["test", "prod"]
         bitrate: ["8k", "16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: tmp/
       STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@@ -2391,7 +2478,7 @@ jobs:
           name: "libstt.tflite.linux.aarch64.zip"
           path: ${{ github.workspace }}/artifacts/libstt.zip
   build-python-LinuxArmv7:
-    name: "LinArmv7|Build python bindings"
+    name: "LinArmv7|Build Python bindings"
     runs-on: ubuntu-20.04
     needs: [ build-lib_LinuxArmv7, swig_Linux, tensorflow_opt-LinuxArmv7 ]
     strategy:
@@ -2447,6 +2534,9 @@ jobs:
         uses: ./.github/actions/numpy_vers
         with:
           pyver: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install -U pip setuptools wheel
       - uses: ./.github/actions/python-build
         with:
           numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
@@ -2456,7 +2546,7 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-armv7.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/native_client/python/dist/*.whl
   build-nodejs-LinuxArmv7:
     name: "LinArmv7|Build NodeJS and ElectronJS"
     runs-on: ubuntu-20.04
@@ -2531,7 +2621,7 @@ jobs:
           name: "stt_intermediate-tflite-armv7.tgz"
           path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz
   build-python-LinuxAarch64:
-    name: "LinAarch64|Build python bindings"
+    name: "LinAarch64|Build Python bindings"
     runs-on: ubuntu-20.04
     needs: [ build-lib_LinuxAarch64, swig_Linux, tensorflow_opt-LinuxAarch64 ]
     strategy:
@@ -2589,6 +2679,9 @@ jobs:
         uses: ./.github/actions/numpy_vers
         with:
           pyver: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install -U pip setuptools wheel
       - uses: ./.github/actions/python-build
         with:
           numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
@@ -2598,7 +2691,7 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: "stt-tflite-${{ matrix.python-version }}-aarch64.whl"
-          path: ${{ github.workspace }}/wheels/*.whl
+          path: ${{ github.workspace }}/native_client/python/dist/*.whl
   build-nodejs-LinuxAarch64:
     name: "LinAarch64|Build NodeJS and ElectronJS"
     runs-on: ubuntu-20.04
@@ -2834,6 +2927,7 @@ jobs:
         nodejs-version: [10, 12, 14, 16]
         models: ["test"]
         bitrate: ["16k"]
+      fail-fast: false
     env:
       CI_TMP_DIR: ${{ github.workspace }}/tmp
       DEBIAN_FRONTEND: "noninteractive"

ci_scripts/build-utils.sh

@@ -11,7 +11,6 @@ do_bazel_build()
 
   bazel ${BAZEL_OUTPUT_USER_ROOT} build \
     -s --explain bazel_explain.log --verbose_explanations \
-    --experimental_strict_action_env \
     --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" \
     -c ${_opt_or_dbg} ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
 

ci_scripts/package-utils.sh

@@ -34,12 +34,18 @@ package_native_client()
     win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so.if.lib"
   fi;
 
+  libsox_lib=""
+  if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then
+    libsox_lib="-C ${stt_dir}/sox-build/lib libsox.so.3"
+  fi
+
   ${TAR} --verbose -cf - \
     --transform='flags=r;s|README.coqui|KenLM_License_Info.txt|' \
     -C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so \
     -C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so \
     -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so \
     ${win_lib} \
+    ${libsox_lib} \
     -C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \
     -C ${stt_dir}/ LICENSE \
     -C ${stt_dir}/native_client/ stt${PLATFORM_EXE_SUFFIX} \
@@ -85,6 +91,7 @@ package_native_client_ndk()
 package_libstt_as_zip()
 {
   tensorflow_dir=${DS_TFDIR}
+  stt_dir=${DS_DSDIR}
   artifacts_dir=${CI_ARTIFACTS_DIR}
   artifact_name=$1
 
@@ -99,8 +106,14 @@ package_libstt_as_zip()
     echo "Please specify artifact name."
   fi;
 
+  libsox_lib=""
+  if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then
+    libsox_lib="${stt_dir}/sox-build/lib/libsox.so.3"
+  fi
+
   ${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" \
     ${tensorflow_dir}/bazel-bin/native_client/libstt.so \
     ${tensorflow_dir}/bazel-bin/native_client/libkenlm.so \
+    ${libsox_lib} \
     ${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so
 }

ci_scripts/tf-build.sh

@@ -8,13 +8,6 @@ source $(dirname $0)/tf-vars.sh
 pushd ${DS_ROOT_TASK}/tensorflow/
     BAZEL_BUILD="bazel ${BAZEL_OUTPUT_USER_ROOT} build -s"
 
-    # Start a bazel process to ensure reliability on Windows and avoid:
-    # FATAL: corrupt installation: file 'c:\builds\tc-workdir\.bazel_cache/install/6b1660721930e9d5f231f7d2a626209b/_embedded_binaries/build-runfiles.exe' missing.
-    bazel ${BAZEL_OUTPUT_USER_ROOT} info
-
-    # Force toolchain sync (useful on macOS ?)
-    bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure
-
     MAYBE_DEBUG=$2
     OPT_OR_DBG="-c opt"
     if [ "${MAYBE_DEBUG}" = "dbg" ]; then
@@ -47,6 +40,4 @@ pushd ${DS_ROOT_TASK}/tensorflow/
         echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB}
         ;;
     esac
-
-    bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown
 popd

ci_scripts/tf-setup.sh

@@ -17,7 +17,7 @@ download()
 {
     fname=`basename $1`
 
-    ${WGET} $1 -O ${DS_ROOT_TASK}/dls/$fname && echo "$2  ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} -
+    ${CURL} -sSL -o ${DS_ROOT_TASK}/dls/$fname $1 && echo "$2  ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} -
 }
 
 # Download stuff
@@ -34,19 +34,15 @@ ls -hal ${DS_ROOT_TASK}/dls/
 
 # Install Bazel in ${DS_ROOT_TASK}/bin
 BAZEL_INSTALL_FILENAME=$(basename "${BAZEL_URL}")
-if [ "${OS}" = "Linux" ]; then
-    BAZEL_INSTALL_FLAGS="--user"
-elif [ "${OS}" = "Darwin" ]; then
-    BAZEL_INSTALL_FLAGS="--bin=${DS_ROOT_TASK}/bin --base=${DS_ROOT_TASK}/.bazel"
-fi;
 mkdir -p ${DS_ROOT_TASK}/bin || true
-pushd ${DS_ROOT_TASK}/bin
+
-    if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
+SUFFIX=""
-        cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel.exe
+if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
-    else
+    SUFFIX=".exe"
-        /bin/bash ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${BAZEL_INSTALL_FLAGS}
+fi
-    fi
+
-popd
+cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel${SUFFIX}
+chmod +x ${DS_ROOT_TASK}/bin/bazel${SUFFIX}
 
 # For debug
 bazel version
@@ -71,8 +67,3 @@ if [ ! -z "${install_android}" ]; then
 fi
 
 mkdir -p ${CI_ARTIFACTS_DIR} || true
-
-
-# Taken from https://www.tensorflow.org/install/source
-# Only future is needed for our builds, as we don't build the Python package
-python -m pip install -U --user future==0.17.1 || true

ci_scripts/tf-vars.sh

@@ -6,8 +6,8 @@ export OS=$(uname)
 if [ "${OS}" = "Linux" ]; then
     export DS_ROOT_TASK=${CI_TASK_DIR}
 
-    BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-linux-x86_64.sh
+    BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-linux-amd64
-    BAZEL_SHA256=7ba815cbac712d061fe728fef958651512ff394b2708e89f79586ec93d1185ed
+    BAZEL_SHA256=4cb534c52cdd47a6223d4596d530e7c9c785438ab3b0a49ff347e991c210b2cd
 
     ANDROID_NDK_URL=https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip
     ANDROID_NDK_SHA256=4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd
@@ -38,8 +38,8 @@ elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
     export TEMP=${CI_TASK_DIR}/tmp/
     export TMP=${CI_TASK_DIR}/tmp/
 
-    BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-windows-x86_64.exe
+    BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-windows-amd64.exe
-    BAZEL_SHA256=776db1f4986dacc3eda143932f00f7529f9ee65c7c1c004414c44aaa6419d0e9
+    BAZEL_SHA256=9a89e6a8cc0a3aea37affcf8c146d8925ffbda1d2290c0c6a845ea81e05de62c
 
     TAR=/usr/bin/tar.exe
 elif [ "${OS}" = "Darwin" ]; then
@@ -52,14 +52,15 @@ elif [ "${OS}" = "Darwin" ]; then
 
     export DS_ROOT_TASK=${CI_TASK_DIR}
 
-    BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-darwin-x86_64.sh
+    BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-darwin-amd64
-    BAZEL_SHA256=5cfa97031b43432b3c742c80e2e01c41c0acdca7ba1052fc8cf1e291271bc9cd
+    BAZEL_SHA256=e485bbf84532d02a60b0eb23c702610b5408df3a199087a4f2b5e0995bbf2d5a
 
     SHA_SUM="shasum -a 256 -c"
     TAR=gtar
 fi;
 
 WGET=${WGET:-"wget"}
+CURL=${CURL:-"curl"}
 TAR=${TAR:-"tar"}
 XZ=${XZ:-"xz -9 -T0"}
 ZIP=${ZIP:-"zip"}
@@ -110,8 +111,8 @@ export GCC_HOST_COMPILER_PATH=/usr/bin/gcc
 
 if [ "${OS}" = "Linux" ]; then
     source /etc/os-release
-    if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then
+    if [ "${ID}" = "debian" -a "${VERSION_ID}" = "9" ]; then
-        export PYTHON_BIN_PATH=/usr/bin/python3
+        export PYTHON_BIN_PATH=/opt/python/cp37-cp37m/bin/python
     fi
 elif [ "${OS}" != "${TC_MSYS_VERSION}" ]; then
     export PYTHON_BIN_PATH=python

native_client/definitions.mk

@@ -1,6 +1,7 @@
 NC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 TARGET    ?= host
+ROOT_DIR  ?= $(abspath $(NC_DIR)/..)
 TFDIR     ?= $(abspath $(NC_DIR)/../tensorflow)
 PREFIX    ?= /usr/local
 SO_SEARCH ?= $(TFDIR)/bazel-bin/
@@ -28,7 +29,7 @@ TOOLCHAIN       :=
 CFLAGS          :=
 CXXFLAGS        :=
 LDFLAGS         :=
-SOX_CFLAGS      := `pkg-config --cflags sox`
+SOX_CFLAGS      := -I$(ROOT_DIR)/sox-build/include
 ifeq ($(OS),Linux)
 MAGIC_LINK_LZMA := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.so | grep lzma | grep '*UND*' | wc -l)
 ifneq ($(MAGIC_LINK_LZMA),0)
@@ -38,8 +39,7 @@ MAGIC_LINK_BZ2  := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.s
 ifneq ($(MAGIC_LINK_BZ2),0)
 MAYBE_LINK_BZ2  := -lbz2
 endif # MAGIC_LINK_BZ2
-SOX_CFLAGS      += -fopenmp
+SOX_LDFLAGS     := -L$(ROOT_DIR)/sox-build/lib -lsox
-SOX_LDFLAGS     := -Wl,-Bstatic `pkg-config --static --libs sox` -lgsm `pkg-config --static --libs libpng | cut -d' ' -f1` -lz -lmagic $(MAYBE_LINK_LZMA) $(MAYBE_LINK_BZ2) -lltdl -Wl,-Bdynamic -ldl
 else ifeq ($(OS),Darwin)
 LIBSOX_PATH             := $(shell echo `pkg-config --libs-only-L sox | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l sox | sed -e 's/^-l//'`.dylib)
 LIBOPUSFILE_PATH        := $(shell echo `pkg-config --libs-only-L opusfile | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l opusfile | sed -e 's/^-l//'`.dylib)
@@ -172,7 +172,7 @@ define copy_missing_libs
     SRC_FILE=$(1); \
     TARGET_LIB_DIR=$(2); \
     MANIFEST_IN=$(3); \
-    echo "Analyzing $$SRC_FILE copying missing libs to $$SRC_FILE"; \
+    echo "Analyzing $$SRC_FILE copying missing libs to $$TARGET_LIB_DIR"; \
     echo "Maybe outputting to $$MANIFEST_IN"; \
     \
     (mkdir $$TARGET_LIB_DIR || true); \
@@ -188,6 +188,7 @@ define copy_missing_libs
         fi; \
     done; \
     \
+    echo "Missing libs = $$missing_libs"; \
     for missing in $$missing_libs; do \
         find $(SO_SEARCH) -type f -name "$$missing" -exec cp {} $$TARGET_LIB_DIR \; ; \
         chmod +w $$TARGET_LIB_DIR/*.so ; \