experiments
/
STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add Android build tasks

This commit is contained in:
Reuben Morais 2021-09-21 16:18:51 +02:00
parent d4091badf9
commit c536d1bd01
17 changed files with 344 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/actions/get_cache_key/action.yml vendored
View File

@ -16,6 +16,7 @@ runs:
steps:
- id: compute_cache_key
run: |
set -xe
JOB=${{ github.job }}
SUBMODULE=$(echo $JOB | cut -d'-' -f1 | cut -d'_' -f1)
FLAVOR=$(echo $JOB | cut -d'-' -f1 | cut -d'_' -f2)

7
.github/actions/setup-tensorflow/action.yml vendored
View File

@ -1,7 +1,12 @@
name: "Setup TensorFlow"
description: "Setup TensorFlow Build"
inputs:
flavor:
description: "Target flavor for setup script (empty/android-armv7/android-arm64)"
required: false
default: ""
runs:
using: "composite"
steps:
- run: ./ci_scripts/tf-setup.sh
- run: ./ci_scripts/tf-setup.sh ${{ inputs.flavor }}
shell: bash

184
.github/workflows/build-and-test.yml vendored
View File

@ -10,6 +10,7 @@ env:
# Shared variables
CI_TASK_DIR: ${{ github.workspace }}
CI_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-14-g4bdd3955115"
# macOS specific
MACOSX_DEPLOYMENT_TARGET: "10.10"
@ -490,7 +491,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -502,6 +502,7 @@ jobs:
- run: |
cd ${{ env.CI_TMP_DIR }}
mkdir ds && cd ds && tar xf ../native_client.tar.xz
ls -lh
- uses: actions/download-artifact@v2
with:
name: "test-model.tflite-${{ matrix.bitrate }}.zip"
@ -530,7 +531,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -577,7 +577,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -631,7 +630,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1294,6 +1292,11 @@ jobs:
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- run: |
# Taken from https://www.tensorflow.org/install/source
# Only future is needed for our builds, as we don't build the Python package
python -m pip install -U --user future==0.17.1 || true
if: needs.tensorflow_opt-macOS.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--darwin-cpu"
@ -1457,7 +1460,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1497,7 +1499,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1542,7 +1543,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1593,7 +1593,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -1777,7 +1776,7 @@ jobs:
rm ${{ needs.tensorflow_opt-Windows.outputs.cache_key }}.tar.xz
- run: |
git status
- run: ./ci_scripts/libstt-build.sh
- run: ./ci_scripts/host-build.sh
- run: ./ci_scripts/package.sh
- uses: actions/upload-artifact@v2
with:
@ -1922,7 +1921,6 @@ jobs:
env:
CI_TMP_DIR: tmp/
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -1976,7 +1974,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2030,7 +2027,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2093,7 +2089,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2206,7 +2201,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -2260,7 +2254,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -2316,7 +2309,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -2368,7 +2360,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- uses: actions/checkout@v2
with:
@ -2422,7 +2413,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -2486,7 +2476,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
steps:
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
@ -3035,7 +3024,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
steps:
- name: "Install QEMU"
@ -3097,7 +3085,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
PIP_EXTRA_INDEX_URL: "https://www.piwheels.org/simple https://lissyx.github.io/deepspeech-python-wheels/"
steps:
@ -3161,7 +3148,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
steps:
- name: "Install QEMU"
@ -3225,7 +3211,6 @@ jobs:
STT_PROD_MODEL: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pb
STT_PROD_MODEL_MMAP: https://github.com/reuben/STT/releases/download/v0.7.0-alpha.3/output_graph.pbmm
STT_TEST_MODEL: ${{ github.workspace }}/tmp/output_graph.pb
EXPECTED_TENSORFLOW_VERSION: "TensorFlow: v2.3.0-6-g23ad988"
SYSTEM_RASPBIAN: ${{ github.workspace }}/chroot-${{ matrix.arch }}
DISPLAY: ":99.0"
steps:
@ -3289,3 +3274,156 @@ jobs:
run: |
cat ${{ env.CI_TMP_DIR }}/xvfb.pid
sudo kill -9 $(cat ${{ env.CI_TMP_DIR }}/xvfb.pid)
# Android jobs
tensorflow_opt-AndroidArmv7:
name: "AndroidArmv7|Check TensorFlow cache"
runs-on: ubuntu-20.04
outputs:
status: ${{ steps.check_artifact_exists.outputs.status }}
cache_key: ${{ steps.get_cache_key.outputs.key }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- id: get_cache_key
uses: ./.github/actions/get_cache_key
with:
extras: "1"
- id: check_artifact_exists
uses: ./.github/actions/check_artifact_exists
with:
name: ${{ steps.get_cache_key.outputs.key }}.tar.xz
build-tensorflow-AndroidArmv7:
name: "AndroidArmv7|Build TensorFlow (opt)"
needs: tensorflow_opt-AndroidArmv7
runs-on: ubuntu-20.04
steps:
- run: true
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'found'
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- name: Use Java 8 instead of Java 11
run: echo "JAVA_HOME=$JAVA_HOME_8_X64" >> $GITHUB_ENV
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
with:
flavor: "--android-armv7"
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--android-armv7"
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- uses: ./.github/actions/package-tensorflow
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
- uses: ./.github/actions/upload-release-asset
with:
name: ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-AndroidArmv7.outputs.status == 'missing'
build-lib_AndroidArmv7:
name: "AndroidArmv7|Build libstt+client"
runs-on: ubuntu-20.04
needs: [ build-tensorflow-AndroidArmv7, tensorflow_opt-AndroidArmv7 ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-AndroidArmv7.outputs.cache_key }}.tar.xz
- uses: ./.github/actions/libstt-build
with:
arch: android-armv7
- run: ./ci_scripts/android-package.sh armeabi-v7a
- uses: actions/upload-artifact@v2
with:
name: "native_client.tflite.android.armv7.tar.xz"
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libstt.tflite.android.armv7.zip"
path: ${{ github.workspace }}/artifacts/libstt.zip
tensorflow_opt-AndroidArm64:
name: "AndroidArm64|Check TensorFlow cache"
runs-on: ubuntu-20.04
outputs:
status: ${{ steps.check_artifact_exists.outputs.status }}
cache_key: ${{ steps.get_cache_key.outputs.key }}
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 1
- id: get_cache_key
uses: ./.github/actions/get_cache_key
with:
extras: "1"
- id: check_artifact_exists
uses: ./.github/actions/check_artifact_exists
with:
name: ${{ steps.get_cache_key.outputs.key }}.tar.xz
build-tensorflow-AndroidArm64:
name: "AndroidArm64|Build TensorFlow (opt)"
needs: tensorflow_opt-AndroidArm64
runs-on: ubuntu-20.04
steps:
- run: true
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'found'
- uses: actions/checkout@v2
with:
fetch-depth: 0
submodules: 'recursive'
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- name: Use Java 8 instead of Java 11
run: echo "JAVA_HOME=$JAVA_HOME_8_X64" >> $GITHUB_ENV
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- uses: ./.github/actions/setup-tensorflow
with:
flavor: "--android-arm64"
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- uses: ./.github/actions/build-tensorflow
with:
flavor: "--android-arm64"
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- uses: ./.github/actions/package-tensorflow
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
- uses: ./.github/actions/upload-release-asset
with:
name: ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/artifacts/home.tar.xz
if: needs.tensorflow_opt-AndroidArm64.outputs.status == 'missing'
build-lib_AndroidArm64:
name: "AndroidArm64|Build libstt+client"
runs-on: ubuntu-20.04
needs: [ build-tensorflow-AndroidArm64, tensorflow_opt-AndroidArm64 ]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- uses: ./.github/actions/check_artifact_exists
with:
name: ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
path: ${{ github.workspace }}/
download: true
- run: |
tar --skip-old-files -xf ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
rm ${{ needs.tensorflow_opt-AndroidArm64.outputs.cache_key }}.tar.xz
- uses: ./.github/actions/libstt-build
with:
arch: android-arm64
- run: ./ci_scripts/android-package.sh arm64-v8a
- uses: actions/upload-artifact@v2
with:
name: "native_client.tflite.android.arm64.tar.xz"
path: ${{ github.workspace }}/artifacts/native_client.tar.xz
- uses: actions/upload-artifact@v2
with:
name: "libstt.tflite.android.arm64.zip"
path: ${{ github.workspace }}/artifacts/libstt.zip

23
ci_scripts/android-arm64-build.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libstt.so
//native_client:generate_scorer_package
"
BAZEL_BUILD_FLAGS="${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=
SYSTEM_RASPBIAN=
do_bazel_build
do_stt_ndk_build "arm64-v8a"

23
ci_scripts/android-armv7-build.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libstt.so
//native_client:generate_scorer_package
"
BAZEL_BUILD_FLAGS="${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=
SYSTEM_RASPBIAN=
do_bazel_build
do_stt_ndk_build "armeabi-v7a"

14
ci_scripts/android-package.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/package-utils.sh
mkdir -p ${CI_ARTIFACTS_DIR} || true
cp ${DS_DSDIR}/tensorflow/bazel*.log ${CI_ARTIFACTS_DIR}/
arm_flavor=$1
package_native_client_ndk "native_client.tar.xz" "${arm_flavor}"

23
ci_scripts/android-x86_64-build.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/all-vars.sh
source $(dirname "$0")/all-utils.sh
source $(dirname "$0")/build-utils.sh
source $(dirname "$0")/tf-vars.sh
BAZEL_TARGETS="
//native_client:libstt.so
//native_client:generate_scorer_package
"
BAZEL_BUILD_FLAGS="${BAZEL_ANDROID_X86_64_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=
SYSTEM_RASPBIAN=
do_bazel_build
do_stt_ndk_build "x86_64"

15
ci_scripts/build-utils.sh
View File

@ -37,3 +37,18 @@ do_stt_binary_build()
EXTRA_LIBS="${EXTRA_LOCAL_LIBS}" \
stt${PLATFORM_EXE_SUFFIX}
}
do_stt_ndk_build()
{
arch_abi=$1
cd ${DS_DSDIR}/native_client/
${ANDROID_NDK_HOME}/ndk-build \
APP_PLATFORM=android-21 \
APP_BUILD_SCRIPT=$(pwd)/Android.mk \
NDK_PROJECT_PATH=$(pwd) \
APP_STL=c++_shared \
TFDIR=${DS_TFDIR} \
TARGET_ARCH_ABI=${arch_abi}
}

9
ci_scripts/package-utils.sh
View File

@ -30,6 +30,10 @@ package_native_client()
win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so.if.lib"
fi;
if [ -f "${tensorflow_dir}/bazel-bin/native_client/libtflitedelegates.so.if.lib" ]; then
win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/native_client/ libtflitedelegates.so.if.lib"
fi;
if [ -f "${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so.if.lib" ]; then
win_lib="$win_lib -C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so.if.lib"
fi;
@ -43,6 +47,7 @@ package_native_client()
--transform='flags=r;s|README.coqui|KenLM_License_Info.txt|' \
-C ${tensorflow_dir}/bazel-bin/native_client/ libstt.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ libkenlm.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ libtflitedelegates.so \
-C ${tensorflow_dir}/bazel-bin/tensorflow/lite/ libtensorflowlite.so \
${win_lib} \
${libsox_lib} \
@ -80,6 +85,9 @@ package_native_client_ndk()
${TAR} --verbose -cf - \
-C ${stt_dir}/native_client/libs/${arch_abi}/ stt \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libstt.so \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libkenlm.so \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libtflitedelegates.so \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libtensorflowlite.so \
-C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \
-C ${stt_dir}/native_client/libs/${arch_abi}/ libc++_shared.so \
-C ${stt_dir}/native_client/ coqui-stt.h \
@ -114,6 +122,7 @@ package_libstt_as_zip()
${ZIP} -r9 --junk-paths "${artifacts_dir}/${artifact_name}" \
${tensorflow_dir}/bazel-bin/native_client/libstt.so \
${tensorflow_dir}/bazel-bin/native_client/libkenlm.so \
${tensorflow_dir}/bazel-bin/native_client/libtflitedelegates.so \
${libsox_lib} \
${tensorflow_dir}/bazel-bin/tensorflow/lite/libtensorflowlite.so
}

13
ci_scripts/tf-vars.sh
View File

@ -151,12 +151,13 @@ export BAZEL_OUTPUT_USER_ROOT
NVCC_COMPUTE="3.5"
BAZEL_ARM_FLAGS="--config=rpi3 --config=rpi3_opt --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV"
BAZEL_ARM_FLAGS="--config=rpi3_opt"
BAZEL_ARM64_FLAGS="--config=rpi3-armv8_opt"
BAZEL_ANDROID_ARM_FLAGS="--config=android_arm"
BAZEL_ANDROID_ARM64_FLAGS="--config=android_arm64"
BAZEL_ANDROID_X86_64_FLAGS="--config=android_x86_64"
BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64"
BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64"
if [ "${OS}" != "${CI_MSYS_VERSION}" ]; then
BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl"

17
native_client/Android.mk
View File

@ -5,10 +5,25 @@ LOCAL_MODULE := stt-prebuilt
LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libstt.so
include $(PREBUILT_SHARED_LIBRARY)
include $(CLEAR_VARS)
LOCAL_MODULE := kenlm-prebuilt
LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libkenlm.so
include $(PREBUILT_SHARED_LIBRARY)
include $(CLEAR_VARS)
LOCAL_MODULE := tensorflowlite-prebuilt
LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/tensorflow/lite/libtensorflowlite.so
include $(PREBUILT_SHARED_LIBRARY)
include $(CLEAR_VARS)
LOCAL_MODULE := tflitedelegates-prebuilt
LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libtflitedelegates.so
include $(PREBUILT_SHARED_LIBRARY)
include $(CLEAR_VARS)
LOCAL_CPP_EXTENSION := .cc .cxx .cpp
LOCAL_MODULE := stt
LOCAL_SRC_FILES := client.cc
LOCAL_SHARED_LIBRARIES := stt-prebuilt
LOCAL_SHARED_LIBRARIES := stt-prebuilt kenlm-prebuilt tensorflowlite-prebuilt tflitedelegates-prebuilt
LOCAL_LDFLAGS := -Wl,--no-as-needed
include $(BUILD_EXECUTABLE)

41
native_client/BUILD
View File

@ -110,12 +110,12 @@ cc_binary(
)
cc_library(
name="kenlm",
name = "kenlm",
hdrs = glob([
"kenlm/lm/*.hh",
"kenlm/util/*.hh",
]),
srcs = ["libkenlm.so"],
srcs = [":libkenlm.so"],
copts = ["-std=c++11"],
defines = ["KENLM_MAX_ORDER=6"],
includes = ["kenlm"],
@ -131,11 +131,10 @@ cc_library(
)
cc_library(
name="tflite",
name = "tflite",
hdrs = [
"//tensorflow/lite:model.h",
"//tensorflow/lite/kernels:register.h",
"//tensorflow/lite/tools/evaluation:utils.h",
] + select({
"//tensorflow:android": [
"//tensorflow/lite/delegates/gpu:delegate.h",
@ -150,6 +149,37 @@ cc_library(
deps = ["//tensorflow/lite:libtensorflowlite.so"],
)
cc_binary(
name = "libtflitedelegates.so",
deps = [
"//tensorflow/lite/tools/evaluation:utils",
],
linkshared = 1,
linkopts = select({
"//tensorflow:ios": [
"-Wl,-install_name,@rpath/libtflitedelegates.so",
],
"//tensorflow:macos": [
"-Wl,-install_name,@rpath/libtflitedelegates.so",
],
"//tensorflow:windows": [],
"//conditions:default": [
"-Wl,-soname,libtflitedelegates.so",
],
}),
)
cc_library(
name = "tflitedelegates",
hdrs = [
"//tensorflow/lite/tools/evaluation:utils.h",
],
deps = [
"//tensorflow/lite/tools/evaluation:utils",
],
srcs = [":libtflitedelegates.so"],
)
cc_library(
name = "coqui_stt_bundle",
srcs = [
@ -184,12 +214,13 @@ cc_library(
# We simply force the linker option manually here as a hacky fix.
"//tensorflow:windows": [
"bazel-out/x64_windows-opt/bin/native_client/libkenlm.so.if.lib",
"bazel-out/x64_windows-opt/bin/native_client/libtflitedelegates.so.if.lib",
"bazel-out/x64_windows-opt/bin/tensorflow/lite/libtensorflowlite.so.if.lib",
],
"//conditions:default": [],
}) + DECODER_LINKOPTS,
includes = DECODER_INCLUDES,
deps = [":kenlm", ":tflite"],
deps = [":kenlm", ":tflite", ":tflitedelegates"],
)
cc_binary(

8
native_client/definitions.mk
View File

@ -21,7 +21,7 @@ endif
STT_BIN := stt$(PLATFORM_EXE_SUFFIX)
CFLAGS_STT := -std=c++11 -o $(STT_BIN)
LINK_STT := -lstt -lkenlm -ltensorflowlite
LINK_STT := -lstt -lkenlm -ltflitedelegates -ltensorflowlite
LINK_PATH_STT := -L${TFDIR}/bazel-bin/native_client -L${TFDIR}/bazel-bin/tensorflow/lite
ifeq ($(TARGET),host)
@ -61,7 +61,7 @@ TOOL_CC := cl.exe
TOOL_CXX := cl.exe
TOOL_LD := link.exe
TOOL_LIBEXE := lib.exe
LINK_STT := $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libstt.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libkenlm.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/tensorflow/lite/libtensorflowlite.so.if.lib")
LINK_STT := $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libstt.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libkenlm.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/native_client/libtflitedelegates.so.if.lib") $(shell cygpath "$(TFDIR)/bazel-bin/tensorflow/lite/libtensorflowlite.so.if.lib")
LINK_PATH_STT :=
CFLAGS_STT := -nologo -Fe$(STT_BIN)
SOX_CFLAGS :=
@ -184,8 +184,8 @@ define copy_missing_libs
if [ "$(OS)" = "Darwin" ]; then \
new_missing="$$( (for f in $$(otool -L $$lib 2>/dev/null | tail -n +2 | awk '{ print $$1 }' | grep -v '$$lib'); do ls -hal $$f; done;) 2>&1 | grep 'No such' | cut -d':' -f2 | xargs basename -a)"; \
missing_libs="$$missing_libs $$new_missing"; \
elif [ "$(OS)" = "${CI_MSYS_VERSION}" ]; then \
missing_libs="libstt.so libkenlm.so libtensorflowlite.so"; \
elif [ "$(OS)" = "${CI_MSYS_VERSION}" ]; then \
missing_libs="libstt.so libkenlm.so libtflitedelegates.so libtensorflowlite.so"; \
else \
missing_libs="$$missing_libs $$($(LDD) $$lib | grep 'not found' | awk '{ print $$1 }')"; \
fi; \

2
native_client/javascript/binding.gyp
View File

@ -27,6 +27,7 @@
"libraries": [
"../../../tensorflow/bazel-bin/native_client/libstt.so.if.lib",
"../../../tensorflow/bazel-bin/native_client/libkenlm.so.if.lib",
"../../../tensorflow/bazel-bin/native_client/libtflitedelegates.so.if.lib",
"../../../tensorflow/bazel-bin/tensorflow/lite/libtensorflowlite.so.if.lib",
],
},
@ -34,6 +35,7 @@
"libraries": [
"../../../tensorflow/bazel-bin/native_client/libstt.so",
"../../../tensorflow/bazel-bin/native_client/libkenlm.so",
"../../../tensorflow/bazel-bin/native_client/libtflitedelegates.so",
"../../../tensorflow/bazel-bin/tensorflow/lite/libtensorflowlite.so",
],
},

4
native_client/tflitemodelstate.cc
View File

@ -1,7 +1,9 @@
#include "tflitemodelstate.h"
#include "tensorflow/lite/string_util.h"
#include "workspace_status.h"
#include "tensorflow/lite/string_util.h"
#include "tensorflow/lite/tools/evaluation/utils.h"
#ifdef __ANDROID__
#include <android/log.h>
#define LOG_TAG "libstt"

1
native_client/tflitemodelstate.h
View File

@ -6,7 +6,6 @@
#include "tensorflow/lite/model.h"
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/tools/evaluation/utils.h"
#include "modelstate.h"

2
tensorflow

@ -1 +1 @@
Subproject commit 182e869fb81d8b82eb4974440ec84d1fd12006f9
Subproject commit 4bdd3955115cc08df61cf94e16a4ea8e0f4847c4