experiments
/
STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #1923 from coqui-ai/tf-libstt-manylinux 

Build TensorFlow+libstt+Python packages in manylinux_2_24 containers
This commit is contained in:
Reuben Morais 2021-08-01 23:56:23 +02:00 committed by GitHub
parent 90a067df49 9d44e2f506
commit 5a5db45c7e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 159 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.github/actions/python-build/action.yml vendored
View File

@ -33,11 +33,6 @@ runs:
- run: |
python3 --version
pip3 --version
python3 -m pip install virtualenv
python3 -m virtualenv stt-build
shell: bash
- run: |
mkdir -p wheels
shell: bash
- run: |
set -xe
@ -45,7 +40,8 @@ runs:
PROJECT_NAME="stt"
OS=$(uname)
if [ "${OS}" = "Linux" ]; then
if [ "${OS}" = "Linux" -a "${{ inputs.target }}" != "host" ]; then
python3 -m venv stt-build
source stt-build/bin/activate
fi
@ -59,14 +55,4 @@ runs:
RASPBIAN=${{ inputs.chroot }} \
SETUP_FLAGS="--project_name ${PROJECT_NAME}" \
bindings-clean bindings
if [ "${OS}" = "Linux" ]; then
deactivate
fi
shell: bash
- run: |
cp native_client/python/dist/*.whl wheels
shell: bash
- run: |
make -C native_client/python/ bindings-clean
shell: bash

131
.github/workflows/build-and-test.yml vendored
View File

@ -111,6 +111,48 @@ jobs:
with:
name: ${{ github.job }}
path: ${{ github.workspace }}/build-static/
libsox3_Linux:
name: "Lin|Build libsox3"
runs-on: ubuntu-20.04
container:
image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
volumes:
- ${{ github.workspace }}:${{ github.workspace }}
steps:
- run: |
curl -sSL https://github.com/coqui-ai/STT/releases/download/v0.10.0-alpha.7/sox-14.4.2.tar.bz2 | tar xjf -
- run: |
mkdir -p sox-build/
- uses: actions/cache@v2
id: sox-build-cache
with:
path: sox-build/
key: sox-1-${{ runner.os }}-${{ env.sox_hash }}
- name: Configure
run: |
cd sox-14.4.2
./configure \
--with-dyn-default --enable-dl-sndfile --enable-dl-amrnb \
--without-magic --without-png --without-ladspa --without-mad \
--without-id3tag --without-lame --without-twolame \
--without-libltdl --disable-openmp --disable-dependency-tracking \
--with-gsm=no --with-lpc10=no --with-oss=no \
--prefix=${{ github.workspace }}/sox-build/
if: steps.sox-build-cache.outputs.cache-hit != 'true'
- name: Make
run: |
cd sox-14.4.2
make -j
if: steps.sox-build-cache.outputs.cache-hit != 'true'
- name: Make install
run: |
cd sox-14.4.2
make install
if: steps.sox-build-cache.outputs.cache-hit != 'true'
- uses: actions/upload-artifact@v2
with:
name: ${{ github.job }}
path: ${{ github.workspace }}/sox-build/
build-ctc-decoder-Linux:
name: "Lin|Build CTC decoder Python package"
needs: [ swig_Linux ]
@ -240,7 +282,7 @@ jobs:
- id: get_cache_key
uses: ./.github/actions/get_cache_key
with:
extras: "3"
extras: "4"
- id: check_artifact_exists
uses: ./.github/actions/check_artifact_exists
with:
@ -249,6 +291,10 @@ jobs:
name: "Lin|Build TensorFlow (opt)"
needs: tensorflow_opt-Linux
runs-on: ubuntu-20.04
container:
image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
volumes:
- ${{ github.workspace }}:${{ github.workspace }}
steps:
- run: true
if: needs.tensorflow_opt-Linux.outputs.status == 'found'
@ -257,11 +303,17 @@ jobs:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
- run: |
sudo apt-get install -y --no-install-recommends pixz
- name: Install dependencies
run: |
apt-get update
apt-get install -y --no-install-recommends xz-utils
if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
if: needs.tensorflow_opt-Linux.outputs.status == 'missing'
- name: Setup venv
run: |
/opt/python/cp37-cp37m/bin/python -m venv /tmp/venv
echo "/tmp/venv/bin" >> $GITHUB_PATH
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--linux-cpu"
@ -276,7 +328,11 @@ jobs:
build-lib_Linux:
name: "Lin|Build libstt+client"
runs-on: ubuntu-20.04
needs: [ build-tensorflow-Linux, tensorflow_opt-Linux ]
needs: [build-tensorflow-Linux, tensorflow_opt-Linux, libsox3_Linux]
container:
image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
volumes:
- ${{ github.workspace }}:${{ github.workspace }}
steps:
- uses: actions/checkout@v2
with:
@ -286,13 +342,21 @@ jobs:
name: ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
download: true
- name: Install dependencies
run: |
apt-get update
apt-get install -y --no-install-recommends xz-utils zip
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-linux.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-Linux.outputs.cache_key }}.tar.xz
- run: |
sudo apt-get install -y --no-install-recommends make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev
- run: |
git status
- name: Setup venv
run: |
/opt/python/cp37-cp37m/bin/python -m venv /tmp/venv
echo "/tmp/venv/bin" >> $GITHUB_PATH
- uses: actions/download-artifact@v2
with:
name: "libsox3_Linux"
path: ${{ github.workspace }}/sox-build/
- uses: ./.github/actions/host-build
- uses: ./.github/actions/package
- uses: actions/upload-artifact@v2
@ -306,10 +370,14 @@ jobs:
build-python-Linux:
name: "Lin|Build Python bindings"
runs-on: ubuntu-20.04
needs: [ build-lib_Linux, swig_Linux ]
needs: [build-lib_Linux, swig_Linux]
strategy:
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
container:
image: quay.io/pypa/manylinux_2_24_x86_64:2021-07-25-cfe8a6c
volumes:
- ${{ github.workspace }}:${{ github.workspace }}
steps:
- uses: actions/checkout@v2
with:
@ -318,7 +386,12 @@ jobs:
with:
name: "native_client.tflite.Linux.tar.xz"
path: ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
- run: |
- name: Install dependencies
run: |
apt-get update
apt-get install -y --no-install-recommends xz-utils
- name: Extract native_client.tar.xz
run: |
cd ${{ github.workspace }}/tensorflow/bazel-bin/native_client/
tar xf native_client.tar.xz
mkdir -p ../tensorflow/lite
@ -334,9 +407,10 @@ jobs:
ls -hal ${{ github.workspace }}/native_client/ds-swig/bin
ln -s ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
chmod +x ${{ github.workspace }}/native_client/ds-swig/bin/ds-swig ${{ github.workspace }}/native_client/ds-swig/bin/swig
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Setup venv
run: |
/opt/_internal/cpython-${{ matrix.python-version }}*/bin/python -m venv /tmp/venv-${{ matrix.python-version }}
echo "/tmp/venv-${{ matrix.python-version }}/bin" >> $GITHUB_PATH
- id: get_numpy
uses: ./.github/actions/numpy_vers
with:
@ -348,7 +422,7 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: "stt-tflite-${{ matrix.python-version }}-Linux.whl"
path: ${{ github.workspace }}/wheels/*.whl
path: ${{ github.workspace }}/native_client/python/dist/*.whl
build-nodejs-Linux:
name: "Lin|Build NodeJS and ElectronJS"
runs-on: ubuntu-20.04
@ -497,6 +571,7 @@ jobs:
nodejs-version: [10, 12, 14, 16]
models: ["test"]
bitrate: ["16k"]
fail-fast: false
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -550,6 +625,7 @@ jobs:
electronjs-version: [5.0.13, 6.1.7, 7.1.8, 8.0.1, 9.2.0, 10.1.0, 11.0.0, 12.0.0]
models: ["test"]
bitrate: ["16k"]
fail-fast: false
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -1083,7 +1159,7 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: "stt-tflite-${{ matrix.python-version }}-macOS.whl"
path: ${{ github.workspace }}/wheels/*.whl
path: ${{ github.workspace }}/native_client/python/dist/*.whl
build-nodejs-macOS:
name: "Mac|Build NodeJS and ElectronJS"
runs-on: macos-10.15
@ -1230,6 +1306,7 @@ jobs:
nodejs-version: [10, 12, 14, 16]
models: ["test"]
bitrate: ["16k"]
fail-fast: false
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -1541,7 +1618,7 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: "stt-tflite-${{ matrix.python-version }}-Windows.whl"
path: ${{ github.workspace }}/wheels/*.whl
path: ${{ github.workspace }}/native_client/python/dist/*.whl
build-nodejs-Windows:
name: "Win|Build NodeJS/ElectronJS"
runs-on: windows-2019
@ -1717,6 +1794,7 @@ jobs:
nodejs-version: [10, 12, 14, 16]
models: ["test"]
bitrate: ["16k"]
fail-fast: false
env:
CI_TMP_DIR: tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -1892,6 +1970,7 @@ jobs:
nodejs-version: [10, 16]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
fail-fast: false
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -1945,6 +2024,7 @@ jobs:
electronjs-version: [5.0.13, 12.0.0]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
fail-fast: false
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -2000,6 +2080,7 @@ jobs:
nodejs-version: [10, 16]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
fail-fast: false
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -2051,6 +2132,7 @@ jobs:
electronjs-version: [5.0.13, 12.0.0]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
fail-fast: false
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -2104,6 +2186,7 @@ jobs:
nodejs-version: [10, 16]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
fail-fast: false
env:
CI_TMP_DIR: tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -2167,6 +2250,7 @@ jobs:
electronjs-version: [5.0.13, 12.0.0]
models: ["test", "prod"]
bitrate: ["8k", "16k"]
fail-fast: false
env:
CI_TMP_DIR: tmp/
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
@ -2391,7 +2475,7 @@ jobs:
name: "libstt.tflite.linux.aarch64.zip"
path: ${{ github.workspace }}/artifacts/libstt.zip
build-python-LinuxArmv7:
name: "LinArmv7|Build python bindings"
name: "LinArmv7|Build Python bindings"
runs-on: ubuntu-20.04
needs: [ build-lib_LinuxArmv7, swig_Linux, tensorflow_opt-LinuxArmv7 ]
strategy:
@ -2447,6 +2531,9 @@ jobs:
uses: ./.github/actions/numpy_vers
with:
pyver: ${{ matrix.python-version }}
- name: Install dependencies
run: |
pip install -U pip setuptools wheel
- uses: ./.github/actions/python-build
with:
numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
@ -2456,7 +2543,7 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: "stt-tflite-${{ matrix.python-version }}-armv7.whl"
path: ${{ github.workspace }}/wheels/*.whl
path: ${{ github.workspace }}/native_client/python/dist/*.whl
build-nodejs-LinuxArmv7:
name: "LinArmv7|Build NodeJS and ElectronJS"
runs-on: ubuntu-20.04
@ -2531,7 +2618,7 @@ jobs:
name: "stt_intermediate-tflite-armv7.tgz"
path: ${{ github.workspace }}/native_client/javascript/stt-*.tgz
build-python-LinuxAarch64:
name: "LinAarch64|Build python bindings"
name: "LinAarch64|Build Python bindings"
runs-on: ubuntu-20.04
needs: [ build-lib_LinuxAarch64, swig_Linux, tensorflow_opt-LinuxAarch64 ]
strategy:
@ -2589,6 +2676,9 @@ jobs:
uses: ./.github/actions/numpy_vers
with:
pyver: ${{ matrix.python-version }}
- name: Install dependencies
run: |
pip install -U pip setuptools wheel
- uses: ./.github/actions/python-build
with:
numpy_build: "${{ steps.get_numpy.outputs.build_version }}"
@ -2598,7 +2688,7 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: "stt-tflite-${{ matrix.python-version }}-aarch64.whl"
path: ${{ github.workspace }}/wheels/*.whl
path: ${{ github.workspace }}/native_client/python/dist/*.whl
build-nodejs-LinuxAarch64:
name: "LinAarch64|Build NodeJS and ElectronJS"
runs-on: ubuntu-20.04
@ -2834,6 +2924,7 @@ jobs:
nodejs-version: [10, 12, 14, 16]
models: ["test"]
bitrate: ["16k"]
fail-fast: false
env:
CI_TMP_DIR: ${{ github.workspace }}/tmp
DEBIAN_FRONTEND: "noninteractive"

1
ci_scripts/build-utils.sh
View File

@ -11,7 +11,6 @@ do_bazel_build()
bazel ${BAZEL_OUTPUT_USER_ROOT} build \
-s --explain bazel_explain.log --verbose_explanations \
--experimental_strict_action_env \
--workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" \
-c ${_opt_or_dbg} ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}

13
ci_scripts/package-utils.sh
View File

@ -34,12 +34,18 @@ package_native_client()
win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so.if.lib"
fi;
libsox_lib=""
if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then
libsox_lib="-C ${stt_dir}/sox-build/lib libsox.so.3"
fi
${TAR} --verbose -cf - \
--transform='flags=r;s|README.coqui|KenLM_License_Info.txt|' \
-C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so \
-C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so \
${win_lib} \
${libsox_lib} \
-C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \
-C ${stt_dir}/ LICENSE \
-C ${stt_dir}/native_client/ stt${PLATFORM_EXE_SUFFIX} \
@ -85,6 +91,7 @@ package_native_client_ndk()
package_libstt_as_zip()
{
tensorflow_dir=${DS_TFDIR}
stt_dir=${DS_DSDIR}
artifacts_dir=${CI_ARTIFACTS_DIR}
artifact_name=$1
@ -99,8 +106,14 @@ package_libstt_as_zip()
echo "Please specify artifact name."
fi;
libsox_lib=""
if [ -f "${stt_dir}/sox-build/lib/libsox.so.3" ]; then
libsox_lib="${stt_dir}/sox-build/lib/libsox.so.3"
fi
${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" \
${tensorflow_dir}/bazel-bin/native_client/libstt.so \
${tensorflow_dir}/bazel-bin/native_client/libkenlm.so \
${libsox_lib} \
${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so
}

9
ci_scripts/tf-build.sh
View File

@ -8,13 +8,6 @@ source $(dirname $0)/tf-vars.sh
pushd ${DS_ROOT_TASK}/tensorflow/
BAZEL_BUILD="bazel ${BAZEL_OUTPUT_USER_ROOT} build -s"
# Start a bazel process to ensure reliability on Windows and avoid:
# FATAL: corrupt installation: file 'c:\builds\tc-workdir\.bazel_cache/install/6b1660721930e9d5f231f7d2a626209b/_embedded_binaries/build-runfiles.exe' missing.
bazel ${BAZEL_OUTPUT_USER_ROOT} info
# Force toolchain sync (useful on macOS ?)
bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure
MAYBE_DEBUG=$2
OPT_OR_DBG="-c opt"
if [ "${MAYBE_DEBUG}" = "dbg" ]; then
@ -47,6 +40,4 @@ pushd ${DS_ROOT_TASK}/tensorflow/
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
esac
bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown
popd

27
ci_scripts/tf-setup.sh
View File

@ -17,7 +17,7 @@ download()
{
fname=`basename $1`
${WGET} $1 -O ${DS_ROOT_TASK}/dls/$fname && echo "$2 ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} -
${CURL} -sSL -o ${DS_ROOT_TASK}/dls/$fname $1 && echo "$2 ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} -
}
# Download stuff
@ -34,19 +34,15 @@ ls -hal ${DS_ROOT_TASK}/dls/
# Install Bazel in ${DS_ROOT_TASK}/bin
BAZEL_INSTALL_FILENAME=$(basename "${BAZEL_URL}")
if [ "${OS}" = "Linux" ]; then
BAZEL_INSTALL_FLAGS="--user"
elif [ "${OS}" = "Darwin" ]; then
BAZEL_INSTALL_FLAGS="--bin=${DS_ROOT_TASK}/bin --base=${DS_ROOT_TASK}/.bazel"
fi;
mkdir -p ${DS_ROOT_TASK}/bin || true
pushd ${DS_ROOT_TASK}/bin
if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel.exe
else
/bin/bash ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${BAZEL_INSTALL_FLAGS}
fi
popd
SUFFIX=""
if [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
SUFFIX=".exe"
fi
cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel${SUFFIX}
chmod +x ${DS_ROOT_TASK}/bin/bazel${SUFFIX}
# For debug
bazel version
@ -71,8 +67,3 @@ if [ ! -z "${install_android}" ]; then
fi
mkdir -p ${CI_ARTIFACTS_DIR} || true
# Taken from https://www.tensorflow.org/install/source
# Only future is needed for our builds, as we don't build the Python package
python -m pip install -U --user future==0.17.1 || true

17
ci_scripts/tf-vars.sh
View File

@ -6,8 +6,8 @@ export OS=$(uname)
if [ "${OS}" = "Linux" ]; then
export DS_ROOT_TASK=${CI_TASK_DIR}
BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-linux-x86_64.sh
BAZEL_SHA256=7ba815cbac712d061fe728fef958651512ff394b2708e89f79586ec93d1185ed
BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-linux-amd64
BAZEL_SHA256=4cb534c52cdd47a6223d4596d530e7c9c785438ab3b0a49ff347e991c210b2cd
ANDROID_NDK_URL=https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip
ANDROID_NDK_SHA256=4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd
@ -38,8 +38,8 @@ elif [ "${OS}" = "${CI_MSYS_VERSION}" ]; then
export TEMP=${CI_TASK_DIR}/tmp/
export TMP=${CI_TASK_DIR}/tmp/
BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-windows-x86_64.exe
BAZEL_SHA256=776db1f4986dacc3eda143932f00f7529f9ee65c7c1c004414c44aaa6419d0e9
BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-windows-amd64.exe
BAZEL_SHA256=9a89e6a8cc0a3aea37affcf8c146d8925ffbda1d2290c0c6a845ea81e05de62c
TAR=/usr/bin/tar.exe
elif [ "${OS}" = "Darwin" ]; then
@ -52,14 +52,15 @@ elif [ "${OS}" = "Darwin" ]; then
export DS_ROOT_TASK=${CI_TASK_DIR}
BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-darwin-x86_64.sh
BAZEL_SHA256=5cfa97031b43432b3c742c80e2e01c41c0acdca7ba1052fc8cf1e291271bc9cd
BAZEL_URL=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-darwin-amd64
BAZEL_SHA256=e485bbf84532d02a60b0eb23c702610b5408df3a199087a4f2b5e0995bbf2d5a
SHA_SUM="shasum -a 256 -c"
TAR=gtar
fi;
WGET=${WGET:-"wget"}
CURL=${CURL:-"curl"}
TAR=${TAR:-"tar"}
XZ=${XZ:-"xz -9 -T0"}
ZIP=${ZIP:-"zip"}
@ -110,8 +111,8 @@ export GCC_HOST_COMPILER_PATH=/usr/bin/gcc
if [ "${OS}" = "Linux" ]; then
source /etc/os-release
if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then
export PYTHON_BIN_PATH=/usr/bin/python3
if [ "${ID}" = "debian" -a "${VERSION_ID}" = "9" ]; then
export PYTHON_BIN_PATH=/opt/python/cp37-cp37m/bin/python
fi
elif [ "${OS}" != "${TC_MSYS_VERSION}" ]; then
export PYTHON_BIN_PATH=python

8
native_client/ctcdecode/Makefile
View File

@ -46,15 +46,15 @@ workspace_status.cc:
# variables over several runs
bindings: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP)
python -m pip install --quiet $(PYTHON_PACKAGES) wheel setuptools
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(SETUP_FLAGS)
find temp_build -type f -name "*.o" -delete
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(SETUP_FLAGS)
rm -rf temp_build
bindings-debug: clean-keep-third-party workspace_status.cc $(DS_SWIG_DEP)
python -m pip install --quiet $(PYTHON_PACKAGES) wheel setuptools
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN_DIR):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(SETUP_FLAGS)
$(GENERATE_DEBUG_SYMS)
find temp_build -type f -name "*.o" -delete
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(SETUP_FLAGS)
rm -rf temp_build

12
native_client/definitions.mk
View File

@ -1,6 +1,7 @@
NC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
TARGET ?= host
ROOT_DIR ?= $(abspath $(NC_DIR)/..)
TFDIR ?= $(abspath $(NC_DIR)/../tensorflow)
PREFIX ?= /usr/local
SO_SEARCH ?= $(TFDIR)/bazel-bin/
@ -28,7 +29,7 @@ TOOLCHAIN :=
CFLAGS :=
CXXFLAGS :=
LDFLAGS :=
SOX_CFLAGS := `pkg-config --cflags sox`
SOX_CFLAGS := -I$(ROOT_DIR)/sox-build/include
ifeq ($(OS),Linux)
MAGIC_LINK_LZMA := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.so | grep lzma | grep '*UND*' | wc -l)
ifneq ($(MAGIC_LINK_LZMA),0)
@ -38,8 +39,7 @@ MAGIC_LINK_BZ2 := $(shell objdump -tTC /usr/lib/`uname -m`-linux-gnu/libmagic.s
ifneq ($(MAGIC_LINK_BZ2),0)
MAYBE_LINK_BZ2 := -lbz2
endif # MAGIC_LINK_BZ2
SOX_CFLAGS += -fopenmp
SOX_LDFLAGS := -Wl,-Bstatic `pkg-config --static --libs sox` -lgsm `pkg-config --static --libs libpng | cut -d' ' -f1` -lz -lmagic $(MAYBE_LINK_LZMA) $(MAYBE_LINK_BZ2) -lltdl -Wl,-Bdynamic -ldl
SOX_LDFLAGS := -L$(ROOT_DIR)/sox-build/lib -lsox
else ifeq ($(OS),Darwin)
LIBSOX_PATH := $(shell echo `pkg-config --libs-only-L sox | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l sox | sed -e 's/^-l//'`.dylib)
LIBOPUSFILE_PATH := $(shell echo `pkg-config --libs-only-L opusfile | sed -e 's/^-L//'`/lib`pkg-config --libs-only-l opusfile | sed -e 's/^-l//'`.dylib)
@ -50,6 +50,9 @@ else
SOX_LDFLAGS := `pkg-config --libs sox`
endif # OS others
PYTHON_PACKAGES := numpy${NUMPY_BUILD_VERSION}
ifeq ($(OS),Linux)
PYTHON_PLATFORM_NAME ?= --plat-name manylinux_2_24_x86_64
endif
endif
ifeq ($(findstring _NT,$(OS)),_NT)
@ -172,7 +175,7 @@ define copy_missing_libs
SRC_FILE=$(1); \
TARGET_LIB_DIR=$(2); \
MANIFEST_IN=$(3); \
echo "Analyzing $$SRC_FILE copying missing libs to $$SRC_FILE"; \
echo "Analyzing $$SRC_FILE copying missing libs to $$TARGET_LIB_DIR"; \
echo "Maybe outputting to $$MANIFEST_IN"; \
\
(mkdir $$TARGET_LIB_DIR || true); \
@ -188,6 +191,7 @@ define copy_missing_libs
fi; \
done; \
\
echo "Missing libs = $$missing_libs"; \
for missing in $$missing_libs; do \
find $(SO_SEARCH) -type f -name "$$missing" -exec cp {} $$TARGET_LIB_DIR \; ; \
chmod +w $$TARGET_LIB_DIR/*.so ; \

3
native_client/javascript/Makefile
View File

@ -63,3 +63,6 @@ npm-pack: clean package.json index.ts npm-dev
stt_wrap.cxx: stt.i ds-swig
$(DS_SWIG_ENV) swig -c++ -javascript -node stt.i
# Hack: disable wrapping of constructors to avoid NodeJS 16.6 ABI compat break
sed -i.bak '/SetCallHandler/d' stt_wrap.cxx
rm stt_wrap.cxx.bak