Fix #3292: Linux debug builds

This commit is contained in:
Alexandre Lissy 2020-09-02 10:52:14 +02:00
parent f20f939ade
commit 86bba80b0e
27 changed files with 534 additions and 25 deletions

View File

@ -11,6 +11,8 @@ training:
deepspeech:
packages_xenial:
apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev'
packages_bionic:
apt: 'apt-get -qq update && apt-get -qq -y install make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python3 python3-dev python-is-python3 zlib1g-dev libbz2-dev liblzma-dev'
kenlm:
packages_xenial:
apt: 'apt-get -qq update && apt-get -qq -y install cmake realpath build-essential libboost-dev wget software-properties-common zlib1g-dev libbz2-dev liblzma-dev libboost-program-options-dev libboost-system-dev libboost-thread-dev libboost-test-dev'
@ -21,12 +23,16 @@ kenlm:
tensorflow:
packages_xenial:
apt: 'apt-get -qq update && apt-get -qq -y install realpath build-essential python-virtualenv python-dev python-pip libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip'
packages_bionic:
apt: 'apt-get -qq update && apt-get -qq -y install coreutils build-essential python3-virtualenv python3-dev python3-pip python-is-python3 libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip'
packages_win:
pacman: 'pacman --noconfirm -S patch unzip tar'
msys64: 'ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64'
java:
packages_xenial:
apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f'
packages_bionic:
apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f'
electronjs:
packages_xenial:
apt: 'libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xvfb'
@ -204,6 +210,47 @@ system:
ios_x86_64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64"
tensorflow_gcc9:
linux_amd64_cpu:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu_gcc9/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu_gcc9"
linux_amd64_cuda:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda_gcc9/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda_gcc9"
tensorflow_dbg:
linux_amd64_cpu:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu"
linux_amd64_cuda:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda"
linux_armv7:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm"
linux_arm64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm64"
darwin_amd64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.osx/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.osx"
android_arm64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-arm64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-arm64"
android_armv7:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-armv7/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-armv7"
win_amd64_cpu:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win"
win_amd64_cuda:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win-cuda/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win-cuda"
ios_arm64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_arm64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_arm64"
ios_x86_64:
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64/artifacts/public/home.tar.xz"
namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64"
username: 'build-user'
homedir:
linux: '/home/build-user'

View File

@ -0,0 +1,21 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
- "tf_android-arm64-dbg"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64-dbg"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64-dbg"
- "index.project.deepspeech.deepspeech.native_client.android-arm64-dbg.${event.head.sha}"
tensorflow: ${system.tensorflow_dbg.android_arm64.url}
scripts:
setup: "taskcluster/tc-true.sh"
build: "taskcluster/android-build-dbg.sh arm64-v8a"
package: "taskcluster/android-package.sh arm64-v8a"
nc_asset_name: "native_client.arm64.cpu.android_dbg.tar.xz"
workerType: "${docker.dsBuild}"
metadata:
name: "DeepSpeech Android ARM64 debug"
description: "Building DeepSpeech for Android ARM64, debug version"

View File

@ -0,0 +1,20 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
- "tf_android-armv7-dbg"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7-dbg"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7-dbg"
- "index.project.deepspeech.deepspeech.native_client.android-armv7-dbg.${event.head.sha}"
tensorflow: ${system.tensorflow_dbg.android_armv7.url}
scripts:
build: "taskcluster/android-build-dbg.sh armeabi-v7a"
package: "taskcluster/android-package.sh armeabi-v7a"
nc_asset_name: "native_client.armv7.cpu.android_dbg.tar.xz"
workerType: "${docker.dsBuild}"
metadata:
name: "DeepSpeech Android ARMv7 debug"
description: "Building DeepSpeech for Android ARMv7, debug version"

View File

@ -0,0 +1,35 @@
#!/bin/bash
set -xe
arm_flavor=$1
source $(dirname "$0")/tc-tests-utils.sh
source $(dirname "$0")/tf_tc-vars.sh
BAZEL_TARGETS="
//native_client:libdeepspeech.so
"
if [ "${arm_flavor}" = "armeabi-v7a" ]; then
LOCAL_ANDROID_FLAGS="${BAZEL_ANDROID_ARM_FLAGS}"
fi
if [ "${arm_flavor}" = "arm64-v8a" ]; then
LOCAL_ANDROID_FLAGS="${BAZEL_ANDROID_ARM64_FLAGS}"
fi
if [ "${arm_flavor}" = "x86_64" ]; then
LOCAL_ANDROID_FLAGS="--config=android --cpu=x86_64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99"
fi
BAZEL_BUILD_FLAGS="--define=runtime=tflite ${LOCAL_ANDROID_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=
SYSTEM_RASPBIAN=
do_bazel_build "dbg"
export EXTRA_LOCAL_CFLAGS="-ggdb"
do_deepspeech_ndk_build "${arm_flavor}"

View File

@ -0,0 +1,20 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
- "tf_android-arm64-dbg"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64-dbg"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64-dbg"
- "index.project.deepspeech.deepspeech.native_client.android-x86_64-dbg.${event.head.sha}"
tensorflow: ${system.tensorflow_dbg.android_arm64.url}
scripts:
build: "taskcluster/android-build-dbg.sh x86_64"
package: "taskcluster/android-package.sh x86_64"
nc_asset_name: "native_client.x86_64.cpu.android_dbg.tar.xz"
workerType: "${docker.dsBuild}"
metadata:
name: "DeepSpeech Android x86_64 debug"
description: "Building DeepSpeech for Android x86_64, debug version"

23
taskcluster/arm64-build-dbg.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
source $(dirname "$0")/tf_tc-vars.sh
BAZEL_TARGETS="
//native_client:libdeepspeech.so
"
BAZEL_BUILD_FLAGS="${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=rpi3-armv8
SYSTEM_RASPBIAN=/tmp/multistrap-armbian64-buster
maybe_install_xldd
do_bazel_build "dbg"
export EXTRA_LOCAL_CFLAGS="-ggdb"
do_deepspeech_binary_build

22
taskcluster/cuda-build-dbg.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
source $(dirname "$0")/tf_tc-vars.sh
BAZEL_TARGETS="
//native_client:libdeepspeech.so
"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}"
BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS}"
SYSTEM_TARGET=host
EXTRA_LOCAL_CFLAGS=""
EXTRA_LOCAL_LDFLAGS="-L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/ -L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/ -lcudart -lcuda"
do_bazel_build "dbg"
export EXTRA_LOCAL_CFLAGS="-ggdb"
do_deepspeech_binary_build

View File

@ -16,6 +16,9 @@ payload:
features:
taskclusterProxy: true
env:
DEBIAN_FRONTEND: "noninteractive"
command:
- "/bin/bash"
- "--login"

27
taskcluster/host-build-dbg.sh Executable file
View File

@ -0,0 +1,27 @@
#!/bin/bash
set -xe
runtime=$1
source $(dirname "$0")/tc-tests-utils.sh
source $(dirname "$0")/tf_tc-vars.sh
BAZEL_TARGETS="
//native_client:libdeepspeech.so
"
if [ "${runtime}" = "tflite" ]; then
BAZEL_BUILD_TFLITE="--define=runtime=tflite"
fi;
BAZEL_BUILD_FLAGS="${BAZEL_BUILD_TFLITE} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=host
do_bazel_build "dbg"
export EXTRA_LOCAL_CFLAGS="-ggdb"
do_deepspeech_binary_build

View File

@ -32,4 +32,3 @@ else
fi
do_deepspeech_nodejs_build

View File

@ -0,0 +1,25 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
- "tf_linux-amd64-cpu_gcc9"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-dbg"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-dbg"
- "index.project.deepspeech.deepspeech.native_client.cpu-dbg.${event.head.sha}"
tensorflow: ${system.tensorflow_gcc9.linux_amd64_cpu.url}
docker_image: "ubuntu:20.04"
system_config:
>
${deepspeech.packages_bionic.apt}
scripts:
setup: "taskcluster/tc-true.sh"
build: "taskcluster/host-build-dbg.sh"
package: "taskcluster/package.sh"
nc_asset_name: "native_client.amd64.cpu.linux_dbg.tar.xz"
workerType: "${docker.tfBuild}"
metadata:
name: "DeepSpeech Linux AMD64 CPU Debug"
description: "Building DeepSpeech for Linux/AMD64, CPU only, debug version"

View File

@ -0,0 +1,26 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
- "tf_linux-amd64-gpu_gcc9"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu-dbg"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu-dbg"
- "index.project.deepspeech.deepspeech.native_client.gpu-dbg.${event.head.sha}"
tensorflow: ${system.tensorflow_gcc9.linux_amd64_cuda.url}
docker_image: "ubuntu:20.04"
system_config:
>
${deepspeech.packages_bionic.apt}
maxRunTime: 14400
scripts:
setup: "taskcluster/tc-true.sh"
build: "taskcluster/cuda-build-dbg.sh"
package: "taskcluster/package.sh"
nc_asset_name: "native_client.amd64.cuda.linux_dbg.tar.xz"
workerType: "${docker.tfBuild}"
metadata:
name: "DeepSpeech Linux AMD64 CUDA debug"
description: "Building DeepSpeech for Linux/AMD64, CUDA-enabled, debug version"

View File

@ -0,0 +1,25 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
- "tf_linux-amd64-cpu_gcc9"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite-dbg"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite-dbg"
- "index.project.deepspeech.deepspeech.native_client.tflite-dbg.${event.head.sha}"
tensorflow: ${system.tensorflow_gcc9.linux_amd64_cpu.url}
docker_image: "ubuntu:20.04"
system_config:
>
${deepspeech.packages_bionic.apt}
scripts:
setup: "taskcluster/tc-true.sh"
build: "taskcluster/host-build-dbg.sh tflite"
package: "taskcluster/package.sh"
nc_asset_name: "native_client.amd64.tflite.linux_dbg.tar.xz"
workerType: "${docker.tfBuild}"
metadata:
name: "DeepSpeech Linux AMD64 TFLite debug"
description: "Building DeepSpeech for Linux/AMD64, TFLite, debug version"

View File

@ -0,0 +1,30 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
- "tf_linux-arm64-cpu-dbg"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64-dbg"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64-dbg"
- "index.project.deepspeech.deepspeech.native_client.arm64-dbg.${event.head.sha}"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>
apt-get -qq -y install gdebi git pixz &&
wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb &&
echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb
system_config:
>
multistrap -d /tmp/multistrap-armbian64-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_armbian64_buster.conf
tensorflow: ${system.tensorflow_dbg.linux_arm64.url}
scripts:
setup: "taskcluster/tc-true.sh"
build: "taskcluster/arm64-build-dbg.sh"
package: "taskcluster/package.sh"
nc_asset_name: "native_client.arm64.cpu.linux_dbg.tar.xz"
workerType: "${docker.dsBuild}"
metadata:
name: "DeepSpeech Linux ARM64 Cortex-A53 CPU debug"
description: "Building DeepSpeech for Linux ARM64 Cortex-A53, CPU only, debug version"

View File

@ -30,10 +30,11 @@ then:
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
image: "ubuntu:16.04"
image: ${build.docker_image}
env:
TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow}
DEBIAN_FRONTEND: "noninteractive"
command:
- "/bin/bash"
@ -44,7 +45,7 @@ then:
extraSystemConfig: { $eval: strip(str(build.system_config)) }
in: >
adduser --system --home ${system.homedir.linux} ${system.username} &&
apt-get -qq update && apt-get -qq -y install ${deepspeech.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} &&
apt-get -qq update && apt-get -qq -y install ${deepspeech.packages_xenial.apt} pixz pkg-config coreutils sudo unzip wget zip && ${extraSystemSetup} &&
cd ${system.homedir.linux}/ &&
echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && cd ~/DeepSpeech/ds && git remote set-url origin ${event.head.repo.url} && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&

View File

@ -0,0 +1,30 @@
build:
template_file: linux-opt-base.tyml
dependencies:
- "swig-linux-amd64"
- "node-gyp-cache"
- "pyenv-linux-amd64"
- "tf_linux-rpi3-cpu-dbg"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm-dbg"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm-dbg"
- "index.project.deepspeech.deepspeech.native_client.arm-dbg.${event.head.sha}"
## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787
system_setup:
>
apt-get -qq -y install gdebi git pixz &&
wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb &&
echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb
system_config:
>
multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf
tensorflow: ${system.tensorflow.linux_armv7.url}
scripts:
setup: "taskcluster/tc-true.sh"
build: "taskcluster/rpi3-build-dbg.sh"
package: "taskcluster/package.sh"
workerType: "${docker.dsBuild}"
nc_asset_name: "native_client.rpi3.cpu.linux_dbg.tar.xz"
metadata:
name: "DeepSpeech Linux RPi3/ARMv7 CPU debug"
description: "Building DeepSpeech for Linux RPi3 ARMv7, CPU only, debug version"

23
taskcluster/rpi3-build-dbg.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
source $(dirname "$0")/tf_tc-vars.sh
BAZEL_TARGETS="
//native_client:libdeepspeech.so
"
BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}"
BAZEL_ENV_FLAGS="TF_NEED_CUDA=0"
SYSTEM_TARGET=rpi3
SYSTEM_RASPBIAN=/tmp/multistrap-raspbian-buster
maybe_install_xldd
do_bazel_build "dbg"
export EXTRA_LOCAL_CFLAGS="-ggdb"
do_deepspeech_binary_build

View File

@ -3,19 +3,25 @@ build:
dependencies:
# Make sure builds are ready
- "linux-arm64-cpu-opt"
- "linux-arm64-cpu-dbg"
- "darwin-amd64-cpu-opt"
- "darwin-amd64-ctc-opt"
- "darwin-amd64-tflite-opt"
- "linux-amd64-cpu-opt"
- "linux-amd64-cpu-dbg"
- "linux-amd64-ctc-opt"
- "linux-amd64-tflite-opt"
- "linux-amd64-tflite-dbg"
- "linux-amd64-gpu-opt"
- "linux-rpi3-cpu-opt"
- "linux-rpi3-cpu-dbg"
- "node-package-gpu"
- "node-package-cpu"
- "node-package-tflite"
- "android-arm64-cpu-opt"
- "android-arm64-cpu-dbg"
- "android-armv7-cpu-opt"
- "android-armv7-cpu-dbg"
- "android-java-opt"
- "win-amd64-cpu-opt"
- "win-amd64-gpu-opt"
@ -64,6 +70,12 @@ build:
- "win-amd64-tflite-opt"
- "ios-x86_64-tflite-opt"
- "ios-arm64-tflite-opt"
- "linux-arm64-cpu-dbg"
- "linux-amd64-cpu-dbg"
- "linux-amd64-tflite-dbg"
- "linux-rpi3-cpu-dbg"
- "android-arm64-cpu-dbg"
- "android-armv7-cpu-dbg"
java_aar:
- "android-java-opt"
nuget:

View File

@ -181,21 +181,26 @@ do_deepspeech_npm_package()
do_bazel_build()
{
local _opt_or_dbg=${1:-"opt"}
cd ${DS_TFDIR}
eval "export ${BAZEL_ENV_FLAGS}"
if [ "${_opt_or_dbg}" = "opt" ]; then
if is_patched_bazel; then
find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T -
fi;
fi;
bazel ${BAZEL_OUTPUT_USER_ROOT} build \
-s --explain bazel_monolithic.log --verbose_explanations --experimental_strict_action_env --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
-s --explain bazel_monolithic.log --verbose_explanations --experimental_strict_action_env --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c ${_opt_or_dbg} ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
if [ "${_opt_or_dbg}" = "opt" ]; then
if is_patched_bazel; then
find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T -
fi;
verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_monolithic.log"
fi;
}
shutdown_bazel()

View File

@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.android_arm64.url}
artifact_namespace: ${system.tensorflow_dbg.android_arm64.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh --android-arm64"
build: "taskcluster/tf_tc-build.sh --android-arm64 dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Android ARM64 debug"
description: "Building TensorFlow for Android ARM64, debug version"

View File

@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.android_armv7.url}
artifact_namespace: ${system.tensorflow_dbg.android_armv7.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh --android-armv7"
build: "taskcluster/tf_tc-build.sh --android-armv7 dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Android ARMv7 debug"
description: "Building TensorFlow for Android ARMv7, debug version"

View File

@ -0,0 +1,18 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_gcc9.linux_amd64_cpu.url}
artifact_namespace: ${system.tensorflow_gcc9.linux_amd64_cpu.namespace}
docker_image: "ubuntu:20.04"
system_config:
>
${tensorflow.packages_bionic.apt} && ${java.packages_bionic.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh"
build: "taskcluster/tf_tc-build.sh --linux-cpu --py3"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Linux AMD64 CPU opt/gcc9"
description: "Building TensorFlow for Linux/AMD64, CPU only, opt/gcc9"

View File

@ -0,0 +1,18 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_gcc9.linux_amd64_cuda.url}
artifact_namespace: ${system.tensorflow_gcc9.linux_amd64_cuda.namespace}
docker_image: "ubuntu:20.04"
system_config:
>
${tensorflow.packages_bionic.apt} && ${java.packages_bionic.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh --linux-cuda"
build: "taskcluster/tf_tc-build.sh --linux-cuda --py3"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Linux AMD64 CUDA opt/gcc9"
description: "Building TensorFlow for Linux/AMD64, CUDA-enabled, opt/gcc9"

View File

@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.linux_arm64.url}
artifact_namespace: ${system.tensorflow_dbg.linux_arm64.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh"
build: "taskcluster/tf_tc-build.sh --linux-arm64 dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Linux ARM64 Cortex-A53 CPU debug"
description: "Building TensorFlow for Linux ARM64 Cortex-A53, CPU only, debug version"

View File

@ -0,0 +1,17 @@
build:
template_file: generic_tc_caching-linux-opt-base.tyml
cache:
artifact_url: ${system.tensorflow_dbg.linux_armv7.url}
artifact_namespace: ${system.tensorflow_dbg.linux_armv7.namespace}
system_config:
>
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
scripts:
setup: "taskcluster/tf_tc-setup.sh"
build: "taskcluster/tf_tc-build.sh --linux-arm dbg"
package: "taskcluster/tf_tc-package.sh"
maxRunTime: 14400
workerType: "${docker.tfBuild}"
metadata:
name: "TensorFlow Linux RPi3/ARMv7 CPU debug"
description: "Building TensorFlow for Linux RPi3 ARMv7, CPU only, debug version"

View File

@ -14,32 +14,36 @@ pushd ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/
# Force toolchain sync (useful on macOS ?)
bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure
OPT_OR_DBG=${2:-opt}
MAYBE_DEBUG=$2
OPT_OR_DBG="-c opt"
if [ "${MAYBE_DEBUG}" = "dbg" ]; then
OPT_OR_DBG="-c dbg"
fi;
case "$1" in
"--linux-cpu"|"--darwin-cpu"|"--windows-cpu")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
;;
"--linux-cuda"|"--windows-cuda")
eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API}
eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API}
;;
"--linux-arm")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--linux-arm64")
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--android-armv7")
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--android-arm64")
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--ios-arm64")
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_IOS_ARM64_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_ARM64_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
"--ios-x86_64")
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB}
echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB}
;;
esac

View File

@ -121,9 +121,18 @@ export GCC_HOST_COMPILER_PATH=/usr/bin/gcc
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
export PYTHON_BIN_PATH=C:/Python36/python.exe
else
if [ "${OS}" = "Linux" ]; then
source /etc/os-release
if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then
export PYTHON_BIN_PATH=/usr/bin/python3
else
export PYTHON_BIN_PATH=/usr/bin/python2.7
fi
else
export PYTHON_BIN_PATH=/usr/bin/python2.7
fi
fi
## Below, define or export some build variables
@ -137,18 +146,16 @@ fi
# See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for targetting specific CPUs
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
CC_OPT_FLAGS="/arch:AVX"
OPT_FLAGS="/arch:AVX"
else
CC_OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx"
OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx"
fi
BAZEL_OPT_FLAGS=""
for flag in ${CC_OPT_FLAGS};
for flag in ${OPT_FLAGS};
do
BAZEL_OPT_FLAGS="${BAZEL_OPT_FLAGS} --copt=${flag}"
done;
export CC_OPT_FLAGS
BAZEL_OUTPUT_CACHE_DIR="${DS_ROOT_TASK}/.bazel_cache/"
BAZEL_OUTPUT_CACHE_INSTANCE="${BAZEL_OUTPUT_CACHE_DIR}/output/"
mkdir -p ${BAZEL_OUTPUT_CACHE_INSTANCE} || true