Merge pull request #1975 from coqui-ai/android-builds

Android builds
This commit is contained in:
Reuben Morais 2021-09-22 13:09:26 +02:00 committed by GitHub
commit 5691d4e053
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 375 additions and 71 deletions

View File

@ -16,6 +16,7 @@ runs:
steps:
- id: compute_cache_key
run: |
set -xe
JOB=${{ github.job }}
SUBMODULE=$(echo $JOB | cut -d'-' -f1 | cut -d'_' -f1)
FLAVOR=$(echo $JOB | cut -d'-' -f1 | cut -d'_' -f2)

View File

@ -1,5 +1,5 @@
name: "Run build lib"
description: "Run build of lib"
name: "Build libstt.so"
description: "Build libstt.so"
inputs:
arch:
description: "Target arch for loading script (host/armv7/aarch64)"

View File

@ -1,7 +1,12 @@
name: "Setup TensorFlow"
description: "Setup TensorFlow Build"
inputs:
flavor:
description: "Target flavor for setup script (empty/android-armv7/android-arm64)"
required: false
default: ""
runs:
using: "composite"
steps:
- run: ./ci_scripts/tf-setup.sh
- run: ./ci_scripts/tf-setup.sh ${{ inputs.flavor }}
shell: bash

View File

@ -10,6 +10,7 @@ env:
# Shared variables
CI_TASK_DIR: ${{ github.workspace }}
CI_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-14-g4bdd3955115"
# macOS specific
MACOSX_DEPLOYMENT_TARGET: "10.10"
@ -357,7 +358,7 @@ jobs:
with:
name: "libsox3_Linux"
path: ${{ github.workspace }}/sox-build/
- uses: ./.github/actions/host-build
- uses: ./.github/actions/libstt-build
- uses: ./.github/actions/package
- uses: actions/upload-artifact@v2
with:
@ -490,7 +491,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -502,6 +502,7 @@ jobs:
- run: |
cd ${{ env.CI_TMP_DIR }}
mkdir ds && cd ds && tar xf ../native_client.tar.xz
ls -lh
- uses: actions/download-artifact@v2
with:
name: "test-model.tflite-${{ matrix.bitrate }}.zip"
@ -530,7 +531,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -577,7 +577,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -631,7 +630,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1310,6 +1308,11 @@ jobs:
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- run: |
# Taken from https://www.tensorflow.org/install/source
# Only future is needed for our builds, as we don't build the Python package
python -m pip install -U --user future==0.17.1 || true
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--darwin-cpu"
@ -1346,7 +1349,7 @@ jobs:
# Taken from https://www.tensorflow.org/install/source
# Only future is needed for our builds, as we don't build the Python package
python -m pip install -U --user future==0.17.1 || true
- uses: ./.github/actions/host-build
- uses: ./.github/actions/libstt-build
- uses: ./.github/actions/package
- uses: actions/upload-artifact@v2
with:
@ -1473,7 +1476,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1513,7 +1515,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1558,7 +1559,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1609,7 +1609,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1938,7 +1937,6 @@ jobs:
env:
CI_TMP_DIR: tmp/
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -1992,7 +1990,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2046,7 +2043,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2109,7 +2105,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2222,7 +2217,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -2276,7 +2270,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -2332,7 +2325,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -2384,7 +2376,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -2438,7 +2429,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2502,7 +2492,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2671,7 +2660,7 @@ jobs:
uses: ./.github/actions/multistrap
with:
arch: armv7
- uses: ./.github/actions/host-build
- uses: ./.github/actions/libstt-build
with:
arch: armv7
- uses: ./.github/actions/package
@ -2708,7 +2697,7 @@ jobs:
uses: ./.github/actions/multistrap
with:
arch: aarch64
- uses: ./.github/actions/host-build
- uses: ./.github/actions/libstt-build
with:
arch: aarch64
- uses: ./.github/actions/package
@ -3051,7 +3040,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
steps:
- name: "Install QEMU"
@ -3113,7 +3101,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple https://lissyx.github.io/deepspeech-python-wheels/"
steps:
@ -3177,7 +3164,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
steps:
- name: "Install QEMU"
@ -3241,7 +3227,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
DISPLAY: ":99.0"
steps:
@ -3305,3 +3290,156 @@ jobs:
run: |
cat ${{ env.CI_TMP_DIR }}/xvfb.pid
sudo kill -9 $(cat ${{ env.CI_TMP_DIR }}/xvfb.pid)
# Android jobs
tensorflow_opt-AndroidArmv7:
name: "AndroidArmv7|Check TensorFlow cache"
runs-on: ubuntu-20.04
outputs:
status: ${{ steps.check_artifact_exists.outputs.status }}
cache_key: ${{ steps.get_cache_key.outputs.key }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- id: get_cache_key
uses: ./.github/actions/get_cache_key
with:
extras: "1"
- id: check_artifact_exists
uses: ./.github/actions/check_artifact_exists
with:
name: ${{ steps.get_cache_key.outputs.key }}.tar.xz
build-tensorflow-AndroidArmv7:
name: "AndroidArmv7|Build TensorFlow (opt)"
needs: tensorflow_opt-AndroidArmv7
runs-on: ubuntu-20.04
steps:
- run: true
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'found'
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- name: Use Java 8 instead of Java 11
run: echo "JAVA_HOME=$JAVA_HOME_8_X64" >> $GITHUB_ENV
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
with:
flavor: "--android-armv7"
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--android-armv7"
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- uses: ./.github/actions/package-tensorflow
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- uses: ./.github/actions/upload-release-asset
with:
name: ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
build-lib_AndroidArmv7:
name: "AndroidArmv7|Build libstt+client"
runs-on: ubuntu-20.04
needs: [ build-tensorflow-AndroidArmv7, tensorflow_opt-AndroidArmv7 ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
- uses: ./.github/actions/libstt-build
with:
arch: android-armv7
- run: ./ci_scripts/android-package.sh armeabi-v7a
- uses: actions/upload-artifact@v2
with:
name: "native_client.tflite.android.armv7.tar.xz"
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libstt.tflite.android.armv7.zip"
path: ${{ github.workspace }}/artifacts/libstt.zip
tensorflow_opt-AndroidArm64:
name: "AndroidArm64|Check TensorFlow cache"
runs-on: ubuntu-20.04
outputs:
status: ${{ steps.check_artifact_exists.outputs.status }}
cache_key: ${{ steps.get_cache_key.outputs.key }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- id: get_cache_key
uses: ./.github/actions/get_cache_key
with:
extras: "1"
- id: check_artifact_exists
uses: ./.github/actions/check_artifact_exists
with:
name: ${{ steps.get_cache_key.outputs.key }}.tar.xz
build-tensorflow-AndroidArm64:
name: "AndroidArm64|Build TensorFlow (opt)"
needs: tensorflow_opt-AndroidArm64
runs-on: ubuntu-20.04
steps:
- run: true
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'found'
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- name: Use Java 8 instead of Java 11
run: echo "JAVA_HOME=$JAVA_HOME_8_X64" >> $GITHUB_ENV
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
with:
flavor: "--android-arm64"
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--android-arm64"
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- uses: ./.github/actions/package-tensorflow
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- uses: ./.github/actions/upload-release-asset
with:
name: ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
build-lib_AndroidArm64:
name: "AndroidArm64|Build libstt+client"
runs-on: ubuntu-20.04
needs: [ build-tensorflow-AndroidArm64, tensorflow_opt-AndroidArm64 ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
- uses: ./.github/actions/libstt-build
with:
arch: android-arm64
- run: ./ci_scripts/android-package.sh arm64-v8a
- uses: actions/upload-artifact@v2
with:
name: "native_client.tflite.android.arm64.tar.xz"
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libstt.tflite.android.arm64.zip"
path: ${{ github.workspace }}/artifacts/libstt.zip

View File

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libstt.so
//native_client:generate_scorer_package
"
BAZEL_BUILD_FLAGS="${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=
SYSTEM_RASPBIAN=
do_bazel_build
do_stt_ndk_build "arm64-v8a"

View File

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libstt.so
//native_client:generate_scorer_package
"
BAZEL_BUILD_FLAGS="${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=
SYSTEM_RASPBIAN=
do_bazel_build
do_stt_ndk_build "armeabi-v7a"

14
ci_scripts/android-package.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/package-utils.sh
mkdir -p ${CI_ARTIFACTS_DIR} || true
cp ${DS_DSDIR}/tensorflow/bazel*.log ${CI_ARTIFACTS_DIR}/
arm_flavor=$1
package_native_client_ndk "native_client.tar.xz" "${arm_flavor}"

View File

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libstt.so
//native_client:generate_scorer_package
"
BAZEL_BUILD_FLAGS="${BAZEL_ANDROID_X86_64_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=
SYSTEM_RASPBIAN=
do_bazel_build
do_stt_ndk_build "x86_64"

View File

@ -37,3 +37,18 @@ do_stt_binary_build()
EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \
stt${PLATFORM_EXE_SUFFIX}
}
do_stt_ndk_build()
{
arch_abi=$1
cd ${DS_DSDIR}/native_client/
${ANDROID_NDK_HOME}/ndk-build \
APP_PLATFORM=android-21 \
APP_BUILD_SCRIPT=$(pwd)/Android.mk \
NDK_PROJECT_PATH=$(pwd) \
APP_STL=c++_shared \
TFDIR=${DS_TFDIR} \
TARGET_ARCH_ABI=${arch_abi}
}

View File

@ -30,6 +30,10 @@ package_native_client()
win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so.if.lib"
fi;
if [ -f "${tensorflow_dir}/bazel-bin/native_client/libtflitedelegates.so.if.lib" ]; then
win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/native_client/ libtflitedelegates.so.if.lib"
fi;
if [ -f "${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so.if.lib" ]; then
win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so.if.lib"
fi;
@ -43,6 +47,7 @@ package_native_client()
--transform='flags=r;s|README.coqui|KenLM_License_Info.txt|' \
-C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ libtflitedelegates.so \
-C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so \
${win_lib} \
${libsox_lib} \
@ -80,6 +85,9 @@ package_native_client_ndk()
${TAR} --verbose -cf - \
-C ${stt_dir}/native_client/libs/${arch_abi}/ stt \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libstt.so \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libkenlm.so \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libtflitedelegates.so \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libtensorflowlite.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libc++_shared.so \
-C ${stt_dir}/native_client/ coqui-stt.h \
@ -114,6 +122,7 @@ package_libstt_as_zip()
${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" \
${tensorflow_dir}/bazel-bin/native_client/libstt.so \
${tensorflow_dir}/bazel-bin/native_client/libkenlm.so \
${tensorflow_dir}/bazel-bin/native_client/libtflitedelegates.so \
${libsox_lib} \
${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so
}

View File

@ -151,12 +151,13 @@ export BAZEL_OUTPUT_USER_ROOT
NVCC_COMPUTE="3.5"
BAZEL_ARM_FLAGS="--config=rpi3 --config=rpi3_opt --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ARM_FLAGS="--config=rpi3_opt"
BAZEL_ARM64_FLAGS="--config=rpi3-armv8_opt"
BAZEL_ANDROID_ARM_FLAGS="--config=android_arm"
BAZEL_ANDROID_ARM64_FLAGS="--config=android_arm64"
BAZEL_ANDROID_X86_64_FLAGS="--config=android_x86_64"
BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64"
BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64"
if [ "${OS}" != "${CI_MSYS_VERSION}" ]; then
BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl"

View File

@ -76,7 +76,7 @@ You can now use Bazel to build the main 🐸STT library, ``libstt.so``. Add ``--
.. code-block::
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c opt --copt=-O3 --copt="-D_GLIBCXX_USE_CXX11_ABI=0" --copt=-fvisibility=hidden //native_client:libstt.so
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" -c opt --copt="-D_GLIBCXX_USE_CXX11_ABI=0" //native_client:libstt.so
The generated binaries will be saved to ``bazel-bin/native_client/``.
@ -90,7 +90,7 @@ Using the example from above you can build the library and that binary at the sa
.. code-block::
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c opt --copt=-O3 --copt="-D_GLIBCXX_USE_CXX11_ABI=0" --copt=-fvisibility=hidden //native_client:libstt.so //native_client:generate_scorer_package
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" -c opt --copt="-D_GLIBCXX_USE_CXX11_ABI=0" //native_client:libstt.so //native_client:generate_scorer_package
The generated binaries will be saved to ``bazel-bin/native_client/``.
@ -188,20 +188,20 @@ RPi3 ARMv7 and LePotato ARM64
We do support cross-compilation. Please refer to our ``coqui-ai/tensorflow`` fork, where we define the following ``--config`` flags:
* ``--config=rpi3`` and ``--config=rpi3_opt`` for Raspbian / ARMv7
* ``--config=rpi3-armv8`` and ``--config=rpi3-armv8_opt`` for ARMBian / ARM64
* ``--config=rpi3_opt`` for Raspbian / ARMv7
* ``--config=rpi3-armv8_opt`` for ARMBian / ARM64
So your command line for ``RPi3`` and ``ARMv7`` should look like:
.. code-block::
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic --config=rpi3 --config=rpi3_opt -c opt --copt=-O3 --copt=-fvisibility=hidden //native_client:libstt.so
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" -c opt --config=rpi3_opt //native_client:libstt.so
And your command line for ``LePotato`` and ``ARM64`` should look like:
.. code-block::
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic --config=rpi3-armv8 --config=rpi3-armv8_opt -c opt --copt=-O3 --copt=-fvisibility=hidden //native_client:libstt.so
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" -c opt --config=rpi3-armv8_opt //native_client:libstt.so
While we test only on RPi3 Raspbian Buster and LePotato ARMBian Buster, anything compatible with ``armv7-a cortex-a53`` or ``armv8-a cortex-a53`` should be fine.
@ -216,10 +216,6 @@ The path of the system tree can be overridden from the default values defined in
Android devices support
-----------------------
We have support for Android relying on TensorFlow Lite, with Java and JNI bindinds. For more details on how to experiment with those, please refer to the section below.
Please refer to TensorFlow documentation on how to setup the environment to build for Android (SDK and NDK required).
Using the library from Android project
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@ -240,36 +236,37 @@ Due to the discontinuation of Bintray JCenter we do not have pre-built Android p
Building ``libstt.so`` for Android
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
You can build the ``libstt.so`` using (ARMv7):
Prerequisites
^^^^^^^^^^^^^
Beyond the general prerequisites listed above, you'll also need the Android-specific dependencies for TensorFlow, namely you'll need to install the `Android SDK <https://developer.android.com>`_ and the `Android NDK version r18b <https://github.com/android/ndk/wiki/Unsupported-Downloads#r18b>`_. After that's done, export the environment variables ``ANDROID_SDK_HOME`` and ``ANDROID_NDK_HOME`` to the corresponding folders where the SDK and NDK were installed. Finally, configure the TensorFlow build and make sure you answer yes when the script asks if you want to set-up an Android build.
Then, you can build the ``libstt.so`` using (ARMv7):
.. code-block::
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic --config=android --config=android_arm --define=runtime=tflite --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 //native_client:libstt.so
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 //native_client:libstt.so
Or (ARM64):
.. code-block::
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic --config=android --config=android_arm64 --define=runtime=tflite --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 //native_client:libstt.so
bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 //native_client:libstt.so
Building ``libstt.aar``
^^^^^^^^^^^^^^^^^^^^^^^
In the unlikely event you have to rebuild the JNI bindings, source code is
available under the ``libstt`` subdirectory. Building depends on shared
object: please ensure to place ``libstt.so`` into the
``libstt/libs/{arm64-v8a,armeabi-v7a,x86_64}/`` matching subdirectories.
In order to build the JNI bindings, source code is available under the ``native_client/java/libstt`` directory. Building the AAR package requires having previously built ``libstt.so`` for all desired architectures and placed the corresponding binaries into the ``native_client/java/libstt/libs/{arm64-v8a,armeabi-v7a,x86_64}/`` subdirectories. If you don't want to build the AAR package for all of ARM64, ARMv7 and x86_64, you can edit the ``native_client/java/libstt/gradle.properties`` file to remove unneeded architectures.
Building the bindings is managed by ``gradle`` and should be limited to issuing
``./gradlew libstt:build``, producing an ``AAR`` package in
``./libstt/build/outputs/aar/``.
Building the bindings is managed by ``gradle`` and can be done by calling ``./gradlew libstt:build`` inside the ``native_client/java`` folder, producing an ``AAR`` package in
``native_client/java/libstt/build/outputs/aar/``.
Please note that you might have to copy the file to a local Maven repository
and adapt file naming (when missing, the error message should states what
filename it expects and where).
Building C++ ``stt`` binary
^^^^^^^^^^^^^^^^^^^^^^^^^^^
Building C++ ``stt`` binary for Android
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Building the ``stt`` binary will happen through ``ndk-build`` (ARMv7):

View File

@ -5,10 +5,25 @@ LOCAL_MODULE := stt-prebuilt
LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libstt.so
include $(PREBUILT_SHARED_LIBRARY)
include $(CLEAR_VARS)
LOCAL_MODULE := kenlm-prebuilt
LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libkenlm.so
include $(PREBUILT_SHARED_LIBRARY)
include $(CLEAR_VARS)
LOCAL_MODULE := tensorflowlite-prebuilt
LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/tensorflow/lite/libtensorflowlite.so
include $(PREBUILT_SHARED_LIBRARY)
include $(CLEAR_VARS)
LOCAL_MODULE := tflitedelegates-prebuilt
LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libtflitedelegates.so
include $(PREBUILT_SHARED_LIBRARY)
include $(CLEAR_VARS)
LOCAL_CPP_EXTENSION := .cc .cxx .cpp
LOCAL_MODULE := stt
LOCAL_SRC_FILES := client.cc
LOCAL_SHARED_LIBRARIES := stt-prebuilt
LOCAL_SHARED_LIBRARIES := stt-prebuilt kenlm-prebuilt tensorflowlite-prebuilt tflitedelegates-prebuilt
LOCAL_LDFLAGS := -Wl,--no-as-needed
include $(BUILD_EXECUTABLE)

View File

@ -110,12 +110,12 @@ cc_binary(
)
cc_library(
name="kenlm",
name = "kenlm",
hdrs = glob([
"kenlm/lm/*.hh",
"kenlm/util/*.hh",
]),
srcs = ["libkenlm.so"],
srcs = [":libkenlm.so"],
copts = ["-std=c++11"],
defines = ["KENLM_MAX_ORDER=6"],
includes = ["kenlm"],
@ -131,12 +131,17 @@ cc_library(
)
cc_library(
name="tflite",
name = "tflite",
hdrs = [
"//tensorflow/lite:model.h",
"//tensorflow/lite/kernels:register.h",
"//tensorflow/lite/tools/evaluation:utils.h",
],
] + select({
"//tensorflow:android": [
"//tensorflow/lite/delegates/gpu:delegate.h",
"//tensorflow/lite/delegates/hexagon:hexagon_delegate.h",
],
"//conditions:default": [],
}),
srcs = [
"//tensorflow/lite:libtensorflowlite.so",
],
@ -144,6 +149,37 @@ cc_library(
deps = ["//tensorflow/lite:libtensorflowlite.so"],
)
cc_binary(
name = "libtflitedelegates.so",
deps = [
"//tensorflow/lite/tools/evaluation:utils",
],
linkshared = 1,
linkopts = select({
"//tensorflow:ios": [
"-Wl,-install_name,@rpath/libtflitedelegates.so",
],
"//tensorflow:macos": [
"-Wl,-install_name,@rpath/libtflitedelegates.so",
],
"//tensorflow:windows": [],
"//conditions:default": [
"-Wl,-soname,libtflitedelegates.so",
],
}),
)
cc_library(
name = "tflitedelegates",
hdrs = [
"//tensorflow/lite/tools/evaluation:utils.h",
],
deps = [
"//tensorflow/lite/tools/evaluation:utils",
],
srcs = [":libtflitedelegates.so"],
)
cc_library(
name = "coqui_stt_bundle",
srcs = [
@ -178,12 +214,13 @@ cc_library(
# We simply force the linker option manually here as a hacky fix.
"//tensorflow:windows": [
"bazel-out/x64_windows-opt/bin/native_client/libkenlm.so.if.lib",
"bazel-out/x64_windows-opt/bin/native_client/libtflitedelegates.so.if.lib",
"bazel-out/x64_windows-opt/bin/tensorflow/lite/libtensorflowlite.so.if.lib",
],
"//conditions:default": [],
}) + DECODER_LINKOPTS,
includes = DECODER_INCLUDES,
deps = [":kenlm", ":tflite"],
deps = [":kenlm", ":tflite", ":tflitedelegates"],
)
cc_binary(

View File

@ -21,7 +21,7 @@ endif
STT_BIN := stt$(PLATFORM_EXE_SUFFIX)
CFLAGS_STT := -std=c++11 -o $(STT_BIN)
LINK_STT := -lstt -lkenlm -ltensorflowlite
LINK_STT := -lstt -lkenlm -ltflitedelegates -ltensorflowlite
LINK_PATH_STT := -L${TFDIR}/bazel-bin/native_client -L${TFDIR}/bazel-bin/tensorflow/lite
ifeq ($(TARGET),host)
@ -61,7 +61,7 @@ TOOL_CC := cl.exe
TOOL_CXX := cl.exe
TOOL_LD := link.exe
TOOL_LIBEXE := lib.exe
LINK_STT := $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libstt.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libkenlm.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/tensorflow/lite/libtensorflowlite.so.if.lib")
LINK_STT := $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libstt.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libkenlm.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libtflitedelegates.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/tensorflow/lite/libtensorflowlite.so.if.lib")
LINK_PATH_STT :=
CFLAGS_STT := -nologo -Fe$(STT_BIN)
SOX_CFLAGS :=
@ -184,8 +184,8 @@ define copy_missing_libs
if [ "$(OS)" = "Darwin" ]; then \
new_missing="$$( (for f in $$(otool -L $$lib 2>/dev/null | tail -n +2 | awk '{ print $$1 }' | grep -v '$$lib'); do ls -hal $$f; done;) 2>&1 | grep 'No such' | cut -d':' -f2 | xargs basename -a)"; \
missing_libs="$$missing_libs $$new_missing"; \
elif [ "$(OS)" = "${CI_MSYS_VERSION}" ]; then \
missing_libs="libstt.so libkenlm.so libtensorflowlite.so"; \
elif [ "$(OS)" = "${CI_MSYS_VERSION}" ]; then \
missing_libs="libstt.so libkenlm.so libtflitedelegates.so libtensorflowlite.so"; \
else \
missing_libs="$$missing_libs $$($(LDD) $$lib | grep 'not found' | awk '{ print $$1 }')"; \
fi; \

View File

@ -27,6 +27,7 @@
"libraries": [
"../../../tensorflow/bazel-bin/native_client/libstt.so.if.lib",
"../../../tensorflow/bazel-bin/native_client/libkenlm.so.if.lib",
"../../../tensorflow/bazel-bin/native_client/libtflitedelegates.so.if.lib",
"../../../tensorflow/bazel-bin/tensorflow/lite/libtensorflowlite.so.if.lib",
],
},
@ -34,6 +35,7 @@
"libraries": [
"../../../tensorflow/bazel-bin/native_client/libstt.so",
"../../../tensorflow/bazel-bin/native_client/libkenlm.so",
"../../../tensorflow/bazel-bin/native_client/libtflitedelegates.so",
"../../../tensorflow/bazel-bin/tensorflow/lite/libtensorflowlite.so",
],
},

View File

@ -1,7 +1,9 @@
#include "tflitemodelstate.h"
#include "tensorflow/lite/string_util.h"
#include "workspace_status.h"
#include "tensorflow/lite/string_util.h"
#include "tensorflow/lite/tools/evaluation/utils.h"
#ifdef __ANDROID__
#include <android/log.h>
#define LOG_TAG "libstt"

View File

@ -6,7 +6,6 @@
#include "tensorflow/lite/model.h"
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/tools/evaluation/utils.h"
#include "modelstate.h"

@ -1 +1 @@
Subproject commit 23ad988fcde60fb01f9533e95004bbc4877a9143
Subproject commit 4bdd3955115cc08df61cf94e16a4ea8e0f4847c4