From 86bba80b0e282e58dee8d14aaf1fb8fba86ca34c Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Wed, 2 Sep 2020 10:52:14 +0200 Subject: [PATCH] Fix #3292: Linux debug builds --- taskcluster/.shared.yml | 47 +++++++++++++++++++ taskcluster/android-arm64-cpu-dbg.yml | 21 +++++++++ taskcluster/android-armv7-cpu-dbg.yml | 20 ++++++++ taskcluster/android-build-dbg.sh | 35 ++++++++++++++ taskcluster/android-x86_64-cpu-dbg.yml | 20 ++++++++ taskcluster/arm64-build-dbg.sh | 23 +++++++++ taskcluster/cuda-build-dbg.sh | 22 +++++++++ .../generic_tc_caching-linux-opt-base.tyml | 3 ++ taskcluster/host-build-dbg.sh | 27 +++++++++++ taskcluster/host-build.sh | 1 - taskcluster/linux-amd64-cpu-dbg.yml | 25 ++++++++++ taskcluster/linux-amd64-gpu-dbg.yml.DISABLED | 26 ++++++++++ taskcluster/linux-amd64-tflite-dbg.yml | 25 ++++++++++ taskcluster/linux-arm64-cpu-dbg.yml | 30 ++++++++++++ taskcluster/linux-opt-base.tyml | 5 +- taskcluster/linux-rpi3-cpu-dbg.yml | 30 ++++++++++++ taskcluster/rpi3-build-dbg.sh | 23 +++++++++ taskcluster/scriptworker-task-github.yml | 12 +++++ taskcluster/tc-build-utils.sh | 19 +++++--- taskcluster/tf_android-arm64-dbg.yml | 17 +++++++ taskcluster/tf_android-armv7-dbg.yml | 17 +++++++ taskcluster/tf_linux-amd64-cpu_gcc9.yml | 18 +++++++ .../tf_linux-amd64-gpu_gcc9.yml.DISABLED | 18 +++++++ taskcluster/tf_linux-arm64-cpu-dbg.yml | 17 +++++++ taskcluster/tf_linux-rpi3-cpu-dbg.yml | 17 +++++++ taskcluster/tf_tc-build.sh | 22 +++++---- taskcluster/tf_tc-vars.sh | 19 +++++--- 27 files changed, 534 insertions(+), 25 deletions(-) create mode 100644 taskcluster/android-arm64-cpu-dbg.yml create mode 100644 taskcluster/android-armv7-cpu-dbg.yml create mode 100755 taskcluster/android-build-dbg.sh create mode 100644 taskcluster/android-x86_64-cpu-dbg.yml create mode 100755 taskcluster/arm64-build-dbg.sh create mode 100755 taskcluster/cuda-build-dbg.sh create mode 100755 taskcluster/host-build-dbg.sh create mode 100644 taskcluster/linux-amd64-cpu-dbg.yml create mode 100644 taskcluster/linux-amd64-gpu-dbg.yml.DISABLED create mode 100644 taskcluster/linux-amd64-tflite-dbg.yml create mode 100644 taskcluster/linux-arm64-cpu-dbg.yml create mode 100644 taskcluster/linux-rpi3-cpu-dbg.yml create mode 100755 taskcluster/rpi3-build-dbg.sh create mode 100644 taskcluster/tf_android-arm64-dbg.yml create mode 100644 taskcluster/tf_android-armv7-dbg.yml create mode 100644 taskcluster/tf_linux-amd64-cpu_gcc9.yml create mode 100644 taskcluster/tf_linux-amd64-gpu_gcc9.yml.DISABLED create mode 100644 taskcluster/tf_linux-arm64-cpu-dbg.yml create mode 100644 taskcluster/tf_linux-rpi3-cpu-dbg.yml diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 718b0d83..611d4c26 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -11,6 +11,8 @@ training: deepspeech: packages_xenial: apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev' + packages_bionic: + apt: 'apt-get -qq update && apt-get -qq -y install make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python3 python3-dev python-is-python3 zlib1g-dev libbz2-dev liblzma-dev' kenlm: packages_xenial: apt: 'apt-get -qq update && apt-get -qq -y install cmake realpath build-essential libboost-dev wget software-properties-common zlib1g-dev libbz2-dev liblzma-dev libboost-program-options-dev libboost-system-dev libboost-thread-dev libboost-test-dev' @@ -21,12 +23,16 @@ kenlm: tensorflow: packages_xenial: apt: 'apt-get -qq update && apt-get -qq -y install realpath build-essential python-virtualenv python-dev python-pip libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip' + packages_bionic: + apt: 'apt-get -qq update && apt-get -qq -y install coreutils build-essential python3-virtualenv python3-dev python3-pip python-is-python3 libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip' packages_win: pacman: 'pacman --noconfirm -S patch unzip tar' msys64: 'ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64' java: packages_xenial: apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f' + packages_bionic: + apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f' electronjs: packages_xenial: apt: 'libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xvfb' @@ -204,6 +210,47 @@ system: ios_x86_64: url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64/artifacts/public/home.tar.xz" namespace: "project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64" + tensorflow_gcc9: + linux_amd64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu_gcc9/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu_gcc9" + linux_amd64_cuda: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda_gcc9/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda_gcc9" + tensorflow_dbg: + linux_amd64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cpu" + linux_amd64_cuda: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.cuda" + linux_armv7: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm" + linux_arm64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.arm64" + darwin_amd64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.osx/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.osx" + android_arm64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-arm64" + android_armv7: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.android-armv7" + win_amd64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win" + win_amd64_cuda: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win-cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.win-cuda" + ios_arm64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_arm64" + ios_x86_64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow_dbg.pip.r2.3.23ad988fcde60fb01f9533e95004bbc4877a9143.0.ios_x86_64" username: 'build-user' homedir: linux: '/home/build-user' diff --git a/taskcluster/android-arm64-cpu-dbg.yml b/taskcluster/android-arm64-cpu-dbg.yml new file mode 100644 index 00000000..0edea48a --- /dev/null +++ b/taskcluster/android-arm64-cpu-dbg.yml @@ -0,0 +1,21 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "swig-linux-amd64" + - "node-gyp-cache" + - "pyenv-linux-amd64" + - "tf_android-arm64-dbg" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64-dbg" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64-dbg" + - "index.project.deepspeech.deepspeech.native_client.android-arm64-dbg.${event.head.sha}" + tensorflow: ${system.tensorflow_dbg.android_arm64.url} + scripts: + setup: "taskcluster/tc-true.sh" + build: "taskcluster/android-build-dbg.sh arm64-v8a" + package: "taskcluster/android-package.sh arm64-v8a" + nc_asset_name: "native_client.arm64.cpu.android_dbg.tar.xz" + workerType: "${docker.dsBuild}" + metadata: + name: "DeepSpeech Android ARM64 debug" + description: "Building DeepSpeech for Android ARM64, debug version" diff --git a/taskcluster/android-armv7-cpu-dbg.yml b/taskcluster/android-armv7-cpu-dbg.yml new file mode 100644 index 00000000..5cb1c581 --- /dev/null +++ b/taskcluster/android-armv7-cpu-dbg.yml @@ -0,0 +1,20 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "swig-linux-amd64" + - "node-gyp-cache" + - "pyenv-linux-amd64" + - "tf_android-armv7-dbg" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7-dbg" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7-dbg" + - "index.project.deepspeech.deepspeech.native_client.android-armv7-dbg.${event.head.sha}" + tensorflow: ${system.tensorflow_dbg.android_armv7.url} + scripts: + build: "taskcluster/android-build-dbg.sh armeabi-v7a" + package: "taskcluster/android-package.sh armeabi-v7a" + nc_asset_name: "native_client.armv7.cpu.android_dbg.tar.xz" + workerType: "${docker.dsBuild}" + metadata: + name: "DeepSpeech Android ARMv7 debug" + description: "Building DeepSpeech for Android ARMv7, debug version" diff --git a/taskcluster/android-build-dbg.sh b/taskcluster/android-build-dbg.sh new file mode 100755 index 00000000..9646bb3d --- /dev/null +++ b/taskcluster/android-build-dbg.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -xe + +arm_flavor=$1 + +source $(dirname "$0")/tc-tests-utils.sh + +source $(dirname "$0")/tf_tc-vars.sh + +BAZEL_TARGETS=" +//native_client:libdeepspeech.so +" + +if [ "${arm_flavor}" = "armeabi-v7a" ]; then + LOCAL_ANDROID_FLAGS="${BAZEL_ANDROID_ARM_FLAGS}" +fi + +if [ "${arm_flavor}" = "arm64-v8a" ]; then + LOCAL_ANDROID_FLAGS="${BAZEL_ANDROID_ARM64_FLAGS}" +fi + +if [ "${arm_flavor}" = "x86_64" ]; then + LOCAL_ANDROID_FLAGS="--config=android --cpu=x86_64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99" +fi + +BAZEL_BUILD_FLAGS="--define=runtime=tflite ${LOCAL_ANDROID_FLAGS} ${BAZEL_EXTRA_FLAGS}" +BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" +SYSTEM_TARGET= +SYSTEM_RASPBIAN= + +do_bazel_build "dbg" + +export EXTRA_LOCAL_CFLAGS="-ggdb" +do_deepspeech_ndk_build "${arm_flavor}" diff --git a/taskcluster/android-x86_64-cpu-dbg.yml b/taskcluster/android-x86_64-cpu-dbg.yml new file mode 100644 index 00000000..92d35c25 --- /dev/null +++ b/taskcluster/android-x86_64-cpu-dbg.yml @@ -0,0 +1,20 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "swig-linux-amd64" + - "node-gyp-cache" + - "pyenv-linux-amd64" + - "tf_android-arm64-dbg" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64-dbg" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64-dbg" + - "index.project.deepspeech.deepspeech.native_client.android-x86_64-dbg.${event.head.sha}" + tensorflow: ${system.tensorflow_dbg.android_arm64.url} + scripts: + build: "taskcluster/android-build-dbg.sh x86_64" + package: "taskcluster/android-package.sh x86_64" + nc_asset_name: "native_client.x86_64.cpu.android_dbg.tar.xz" + workerType: "${docker.dsBuild}" + metadata: + name: "DeepSpeech Android x86_64 debug" + description: "Building DeepSpeech for Android x86_64, debug version" diff --git a/taskcluster/arm64-build-dbg.sh b/taskcluster/arm64-build-dbg.sh new file mode 100755 index 00000000..20d4cd87 --- /dev/null +++ b/taskcluster/arm64-build-dbg.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +source $(dirname "$0")/tf_tc-vars.sh + +BAZEL_TARGETS=" +//native_client:libdeepspeech.so +" + +BAZEL_BUILD_FLAGS="${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS}" +BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" +SYSTEM_TARGET=rpi3-armv8 +SYSTEM_RASPBIAN=/tmp/multistrap-armbian64-buster + +maybe_install_xldd + +do_bazel_build "dbg" + +export EXTRA_LOCAL_CFLAGS="-ggdb" +do_deepspeech_binary_build diff --git a/taskcluster/cuda-build-dbg.sh b/taskcluster/cuda-build-dbg.sh new file mode 100755 index 00000000..e5e44d78 --- /dev/null +++ b/taskcluster/cuda-build-dbg.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +source $(dirname "$0")/tf_tc-vars.sh + +BAZEL_TARGETS=" +//native_client:libdeepspeech.so +" + +BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}" +BAZEL_BUILD_FLAGS="${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS}" +SYSTEM_TARGET=host +EXTRA_LOCAL_CFLAGS="" +EXTRA_LOCAL_LDFLAGS="-L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/ -L${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/ -lcudart -lcuda" + +do_bazel_build "dbg" + +export EXTRA_LOCAL_CFLAGS="-ggdb" +do_deepspeech_binary_build diff --git a/taskcluster/generic_tc_caching-linux-opt-base.tyml b/taskcluster/generic_tc_caching-linux-opt-base.tyml index 3e9cbded..516ad132 100644 --- a/taskcluster/generic_tc_caching-linux-opt-base.tyml +++ b/taskcluster/generic_tc_caching-linux-opt-base.tyml @@ -16,6 +16,9 @@ payload: features: taskclusterProxy: true + env: + DEBIAN_FRONTEND: "noninteractive" + command: - "/bin/bash" - "--login" diff --git a/taskcluster/host-build-dbg.sh b/taskcluster/host-build-dbg.sh new file mode 100755 index 00000000..2ced0313 --- /dev/null +++ b/taskcluster/host-build-dbg.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -xe + +runtime=$1 + +source $(dirname "$0")/tc-tests-utils.sh + +source $(dirname "$0")/tf_tc-vars.sh + +BAZEL_TARGETS=" +//native_client:libdeepspeech.so +" + +if [ "${runtime}" = "tflite" ]; then + BAZEL_BUILD_TFLITE="--define=runtime=tflite" +fi; + +BAZEL_BUILD_FLAGS="${BAZEL_BUILD_TFLITE} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS}" + +BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" +SYSTEM_TARGET=host + +do_bazel_build "dbg" + +export EXTRA_LOCAL_CFLAGS="-ggdb" +do_deepspeech_binary_build diff --git a/taskcluster/host-build.sh b/taskcluster/host-build.sh index 9ff3648c..84f07302 100755 --- a/taskcluster/host-build.sh +++ b/taskcluster/host-build.sh @@ -32,4 +32,3 @@ else fi do_deepspeech_nodejs_build - diff --git a/taskcluster/linux-amd64-cpu-dbg.yml b/taskcluster/linux-amd64-cpu-dbg.yml new file mode 100644 index 00000000..9c97429e --- /dev/null +++ b/taskcluster/linux-amd64-cpu-dbg.yml @@ -0,0 +1,25 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "swig-linux-amd64" + - "node-gyp-cache" + - "pyenv-linux-amd64" + - "tf_linux-amd64-cpu_gcc9" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-dbg" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-dbg" + - "index.project.deepspeech.deepspeech.native_client.cpu-dbg.${event.head.sha}" + tensorflow: ${system.tensorflow_gcc9.linux_amd64_cpu.url} + docker_image: "ubuntu:20.04" + system_config: + > + ${deepspeech.packages_bionic.apt} + scripts: + setup: "taskcluster/tc-true.sh" + build: "taskcluster/host-build-dbg.sh" + package: "taskcluster/package.sh" + nc_asset_name: "native_client.amd64.cpu.linux_dbg.tar.xz" + workerType: "${docker.tfBuild}" + metadata: + name: "DeepSpeech Linux AMD64 CPU Debug" + description: "Building DeepSpeech for Linux/AMD64, CPU only, debug version" diff --git a/taskcluster/linux-amd64-gpu-dbg.yml.DISABLED b/taskcluster/linux-amd64-gpu-dbg.yml.DISABLED new file mode 100644 index 00000000..f2744c87 --- /dev/null +++ b/taskcluster/linux-amd64-gpu-dbg.yml.DISABLED @@ -0,0 +1,26 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "swig-linux-amd64" + - "node-gyp-cache" + - "pyenv-linux-amd64" + - "tf_linux-amd64-gpu_gcc9" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu-dbg" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu-dbg" + - "index.project.deepspeech.deepspeech.native_client.gpu-dbg.${event.head.sha}" + tensorflow: ${system.tensorflow_gcc9.linux_amd64_cuda.url} + docker_image: "ubuntu:20.04" + system_config: + > + ${deepspeech.packages_bionic.apt} + maxRunTime: 14400 + scripts: + setup: "taskcluster/tc-true.sh" + build: "taskcluster/cuda-build-dbg.sh" + package: "taskcluster/package.sh" + nc_asset_name: "native_client.amd64.cuda.linux_dbg.tar.xz" + workerType: "${docker.tfBuild}" + metadata: + name: "DeepSpeech Linux AMD64 CUDA debug" + description: "Building DeepSpeech for Linux/AMD64, CUDA-enabled, debug version" diff --git a/taskcluster/linux-amd64-tflite-dbg.yml b/taskcluster/linux-amd64-tflite-dbg.yml new file mode 100644 index 00000000..80dfaa77 --- /dev/null +++ b/taskcluster/linux-amd64-tflite-dbg.yml @@ -0,0 +1,25 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "swig-linux-amd64" + - "node-gyp-cache" + - "pyenv-linux-amd64" + - "tf_linux-amd64-cpu_gcc9" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite-dbg" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite-dbg" + - "index.project.deepspeech.deepspeech.native_client.tflite-dbg.${event.head.sha}" + tensorflow: ${system.tensorflow_gcc9.linux_amd64_cpu.url} + docker_image: "ubuntu:20.04" + system_config: + > + ${deepspeech.packages_bionic.apt} + scripts: + setup: "taskcluster/tc-true.sh" + build: "taskcluster/host-build-dbg.sh tflite" + package: "taskcluster/package.sh" + nc_asset_name: "native_client.amd64.tflite.linux_dbg.tar.xz" + workerType: "${docker.tfBuild}" + metadata: + name: "DeepSpeech Linux AMD64 TFLite debug" + description: "Building DeepSpeech for Linux/AMD64, TFLite, debug version" diff --git a/taskcluster/linux-arm64-cpu-dbg.yml b/taskcluster/linux-arm64-cpu-dbg.yml new file mode 100644 index 00000000..33e3ebc7 --- /dev/null +++ b/taskcluster/linux-arm64-cpu-dbg.yml @@ -0,0 +1,30 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "swig-linux-amd64" + - "node-gyp-cache" + - "pyenv-linux-amd64" + - "tf_linux-arm64-cpu-dbg" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64-dbg" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64-dbg" + - "index.project.deepspeech.deepspeech.native_client.arm64-dbg.${event.head.sha}" + ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 + system_setup: + > + apt-get -qq -y install gdebi git pixz && + wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && + echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb + system_config: + > + multistrap -d /tmp/multistrap-armbian64-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_armbian64_buster.conf + tensorflow: ${system.tensorflow_dbg.linux_arm64.url} + scripts: + setup: "taskcluster/tc-true.sh" + build: "taskcluster/arm64-build-dbg.sh" + package: "taskcluster/package.sh" + nc_asset_name: "native_client.arm64.cpu.linux_dbg.tar.xz" + workerType: "${docker.dsBuild}" + metadata: + name: "DeepSpeech Linux ARM64 Cortex-A53 CPU debug" + description: "Building DeepSpeech for Linux ARM64 Cortex-A53, CPU only, debug version" diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index e84d738e..f7d4f7a2 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -30,10 +30,11 @@ then: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "ubuntu:16.04" + image: ${build.docker_image} env: TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} + DEBIAN_FRONTEND: "noninteractive" command: - "/bin/bash" @@ -44,7 +45,7 @@ then: extraSystemConfig: { $eval: strip(str(build.system_config)) } in: > adduser --system --home ${system.homedir.linux} ${system.username} && - apt-get -qq update && apt-get -qq -y install ${deepspeech.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} && + apt-get -qq update && apt-get -qq -y install ${deepspeech.packages_xenial.apt} pixz pkg-config coreutils sudo unzip wget zip && ${extraSystemSetup} && cd ${system.homedir.linux}/ && echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && cd ~/DeepSpeech/ds && git remote set-url origin ${event.head.repo.url} && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && diff --git a/taskcluster/linux-rpi3-cpu-dbg.yml b/taskcluster/linux-rpi3-cpu-dbg.yml new file mode 100644 index 00000000..bf153d6e --- /dev/null +++ b/taskcluster/linux-rpi3-cpu-dbg.yml @@ -0,0 +1,30 @@ +build: + template_file: linux-opt-base.tyml + dependencies: + - "swig-linux-amd64" + - "node-gyp-cache" + - "pyenv-linux-amd64" + - "tf_linux-rpi3-cpu-dbg" + routes: + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm-dbg" + - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm-dbg" + - "index.project.deepspeech.deepspeech.native_client.arm-dbg.${event.head.sha}" + ## multistrap 2.2.0-ubuntu1 is broken in 14.04: https://bugs.launchpad.net/ubuntu/+source/multistrap/+bug/1313787 + system_setup: + > + apt-get -qq -y install gdebi git pixz && + wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && + echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb + system_config: + > + multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf + tensorflow: ${system.tensorflow.linux_armv7.url} + scripts: + setup: "taskcluster/tc-true.sh" + build: "taskcluster/rpi3-build-dbg.sh" + package: "taskcluster/package.sh" + workerType: "${docker.dsBuild}" + nc_asset_name: "native_client.rpi3.cpu.linux_dbg.tar.xz" + metadata: + name: "DeepSpeech Linux RPi3/ARMv7 CPU debug" + description: "Building DeepSpeech for Linux RPi3 ARMv7, CPU only, debug version" diff --git a/taskcluster/rpi3-build-dbg.sh b/taskcluster/rpi3-build-dbg.sh new file mode 100755 index 00000000..c0652eae --- /dev/null +++ b/taskcluster/rpi3-build-dbg.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -xe + +source $(dirname "$0")/tc-tests-utils.sh + +source $(dirname "$0")/tf_tc-vars.sh + +BAZEL_TARGETS=" +//native_client:libdeepspeech.so +" + +BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}" +BAZEL_ENV_FLAGS="TF_NEED_CUDA=0" +SYSTEM_TARGET=rpi3 +SYSTEM_RASPBIAN=/tmp/multistrap-raspbian-buster + +maybe_install_xldd + +do_bazel_build "dbg" + +export EXTRA_LOCAL_CFLAGS="-ggdb" +do_deepspeech_binary_build diff --git a/taskcluster/scriptworker-task-github.yml b/taskcluster/scriptworker-task-github.yml index 74388b52..17cee7a8 100644 --- a/taskcluster/scriptworker-task-github.yml +++ b/taskcluster/scriptworker-task-github.yml @@ -3,19 +3,25 @@ build: dependencies: # Make sure builds are ready - "linux-arm64-cpu-opt" + - "linux-arm64-cpu-dbg" - "darwin-amd64-cpu-opt" - "darwin-amd64-ctc-opt" - "darwin-amd64-tflite-opt" - "linux-amd64-cpu-opt" + - "linux-amd64-cpu-dbg" - "linux-amd64-ctc-opt" - "linux-amd64-tflite-opt" + - "linux-amd64-tflite-dbg" - "linux-amd64-gpu-opt" - "linux-rpi3-cpu-opt" + - "linux-rpi3-cpu-dbg" - "node-package-gpu" - "node-package-cpu" - "node-package-tflite" - "android-arm64-cpu-opt" + - "android-arm64-cpu-dbg" - "android-armv7-cpu-opt" + - "android-armv7-cpu-dbg" - "android-java-opt" - "win-amd64-cpu-opt" - "win-amd64-gpu-opt" @@ -64,6 +70,12 @@ build: - "win-amd64-tflite-opt" - "ios-x86_64-tflite-opt" - "ios-arm64-tflite-opt" + - "linux-arm64-cpu-dbg" + - "linux-amd64-cpu-dbg" + - "linux-amd64-tflite-dbg" + - "linux-rpi3-cpu-dbg" + - "android-arm64-cpu-dbg" + - "android-armv7-cpu-dbg" java_aar: - "android-java-opt" nuget: diff --git a/taskcluster/tc-build-utils.sh b/taskcluster/tc-build-utils.sh index 693b5edc..b1077ada 100755 --- a/taskcluster/tc-build-utils.sh +++ b/taskcluster/tc-build-utils.sh @@ -181,21 +181,26 @@ do_deepspeech_npm_package() do_bazel_build() { + local _opt_or_dbg=${1:-"opt"} + cd ${DS_TFDIR} eval "export ${BAZEL_ENV_FLAGS}" - if is_patched_bazel; then - find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T - + if [ "${_opt_or_dbg}" = "opt" ]; then + if is_patched_bazel; then + find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T - + fi; fi; bazel ${BAZEL_OUTPUT_USER_ROOT} build \ - -s --explain bazel_monolithic.log --verbose_explanations --experimental_strict_action_env --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS} + -s --explain bazel_monolithic.log --verbose_explanations --experimental_strict_action_env --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c ${_opt_or_dbg} ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS} - if is_patched_bazel; then - find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T - + if [ "${_opt_or_dbg}" = "opt" ]; then + if is_patched_bazel; then + find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T - + fi; + verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_monolithic.log" fi; - - verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_monolithic.log" } shutdown_bazel() diff --git a/taskcluster/tf_android-arm64-dbg.yml b/taskcluster/tf_android-arm64-dbg.yml new file mode 100644 index 00000000..5aa582fc --- /dev/null +++ b/taskcluster/tf_android-arm64-dbg.yml @@ -0,0 +1,17 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow_dbg.android_arm64.url} + artifact_namespace: ${system.tensorflow_dbg.android_arm64.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh --android-arm64" + build: "taskcluster/tf_tc-build.sh --android-arm64 dbg" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + workerType: "${docker.tfBuild}" + metadata: + name: "TensorFlow Android ARM64 debug" + description: "Building TensorFlow for Android ARM64, debug version" diff --git a/taskcluster/tf_android-armv7-dbg.yml b/taskcluster/tf_android-armv7-dbg.yml new file mode 100644 index 00000000..43378f31 --- /dev/null +++ b/taskcluster/tf_android-armv7-dbg.yml @@ -0,0 +1,17 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow_dbg.android_armv7.url} + artifact_namespace: ${system.tensorflow_dbg.android_armv7.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh --android-armv7" + build: "taskcluster/tf_tc-build.sh --android-armv7 dbg" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + workerType: "${docker.tfBuild}" + metadata: + name: "TensorFlow Android ARMv7 debug" + description: "Building TensorFlow for Android ARMv7, debug version" diff --git a/taskcluster/tf_linux-amd64-cpu_gcc9.yml b/taskcluster/tf_linux-amd64-cpu_gcc9.yml new file mode 100644 index 00000000..c92f9932 --- /dev/null +++ b/taskcluster/tf_linux-amd64-cpu_gcc9.yml @@ -0,0 +1,18 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow_gcc9.linux_amd64_cpu.url} + artifact_namespace: ${system.tensorflow_gcc9.linux_amd64_cpu.namespace} + docker_image: "ubuntu:20.04" + system_config: + > + ${tensorflow.packages_bionic.apt} && ${java.packages_bionic.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --linux-cpu --py3" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + workerType: "${docker.tfBuild}" + metadata: + name: "TensorFlow Linux AMD64 CPU opt/gcc9" + description: "Building TensorFlow for Linux/AMD64, CPU only, opt/gcc9" diff --git a/taskcluster/tf_linux-amd64-gpu_gcc9.yml.DISABLED b/taskcluster/tf_linux-amd64-gpu_gcc9.yml.DISABLED new file mode 100644 index 00000000..219a7c98 --- /dev/null +++ b/taskcluster/tf_linux-amd64-gpu_gcc9.yml.DISABLED @@ -0,0 +1,18 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow_gcc9.linux_amd64_cuda.url} + artifact_namespace: ${system.tensorflow_gcc9.linux_amd64_cuda.namespace} + docker_image: "ubuntu:20.04" + system_config: + > + ${tensorflow.packages_bionic.apt} && ${java.packages_bionic.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh --linux-cuda" + build: "taskcluster/tf_tc-build.sh --linux-cuda --py3" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + workerType: "${docker.tfBuild}" + metadata: + name: "TensorFlow Linux AMD64 CUDA opt/gcc9" + description: "Building TensorFlow for Linux/AMD64, CUDA-enabled, opt/gcc9" diff --git a/taskcluster/tf_linux-arm64-cpu-dbg.yml b/taskcluster/tf_linux-arm64-cpu-dbg.yml new file mode 100644 index 00000000..2e01f111 --- /dev/null +++ b/taskcluster/tf_linux-arm64-cpu-dbg.yml @@ -0,0 +1,17 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow_dbg.linux_arm64.url} + artifact_namespace: ${system.tensorflow_dbg.linux_arm64.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --linux-arm64 dbg" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + workerType: "${docker.tfBuild}" + metadata: + name: "TensorFlow Linux ARM64 Cortex-A53 CPU debug" + description: "Building TensorFlow for Linux ARM64 Cortex-A53, CPU only, debug version" diff --git a/taskcluster/tf_linux-rpi3-cpu-dbg.yml b/taskcluster/tf_linux-rpi3-cpu-dbg.yml new file mode 100644 index 00000000..a1772a98 --- /dev/null +++ b/taskcluster/tf_linux-rpi3-cpu-dbg.yml @@ -0,0 +1,17 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow_dbg.linux_armv7.url} + artifact_namespace: ${system.tensorflow_dbg.linux_armv7.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --linux-arm dbg" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + workerType: "${docker.tfBuild}" + metadata: + name: "TensorFlow Linux RPi3/ARMv7 CPU debug" + description: "Building TensorFlow for Linux RPi3 ARMv7, CPU only, debug version" diff --git a/taskcluster/tf_tc-build.sh b/taskcluster/tf_tc-build.sh index 93287bd5..375b3d1c 100755 --- a/taskcluster/tf_tc-build.sh +++ b/taskcluster/tf_tc-build.sh @@ -14,32 +14,36 @@ pushd ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/ # Force toolchain sync (useful on macOS ?) bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure - OPT_OR_DBG=${2:-opt} + MAYBE_DEBUG=$2 + OPT_OR_DBG="-c opt" + if [ "${MAYBE_DEBUG}" = "dbg" ]; then + OPT_OR_DBG="-c dbg" + fi; case "$1" in "--linux-cpu"|"--darwin-cpu"|"--windows-cpu") - echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB} + echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB} ;; "--linux-cuda"|"--windows-cuda") - eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API} + eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ;; "--linux-arm") - echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} ;; "--linux-arm64") - echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} ;; "--android-armv7") - echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} ;; "--android-arm64") - echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} ;; "--ios-arm64") - echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_IOS_ARM64_FLAGS} ${BUILD_TARGET_LITE_LIB} + echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_ARM64_FLAGS} ${BUILD_TARGET_LITE_LIB} ;; "--ios-x86_64") - echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} -c ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB} + echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} ${OPT_OR_DBG} ${BAZEL_IOS_X86_64_FLAGS} ${BUILD_TARGET_LITE_LIB} ;; esac diff --git a/taskcluster/tf_tc-vars.sh b/taskcluster/tf_tc-vars.sh index 4e75931e..4a0e1dc7 100755 --- a/taskcluster/tf_tc-vars.sh +++ b/taskcluster/tf_tc-vars.sh @@ -122,7 +122,16 @@ export GCC_HOST_COMPILER_PATH=/usr/bin/gcc if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then export PYTHON_BIN_PATH=C:/Python36/python.exe else - export PYTHON_BIN_PATH=/usr/bin/python2.7 + if [ "${OS}" = "Linux" ]; then + source /etc/os-release + if [ "${ID}" = "ubuntu" -a "${VERSION_ID}" = "20.04" ]; then + export PYTHON_BIN_PATH=/usr/bin/python3 + else + export PYTHON_BIN_PATH=/usr/bin/python2.7 + fi + else + export PYTHON_BIN_PATH=/usr/bin/python2.7 + fi fi ## Below, define or export some build variables @@ -137,18 +146,16 @@ fi # See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for targetting specific CPUs if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then - CC_OPT_FLAGS="/arch:AVX" + OPT_FLAGS="/arch:AVX" else - CC_OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx" + OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx" fi BAZEL_OPT_FLAGS="" -for flag in ${CC_OPT_FLAGS}; +for flag in ${OPT_FLAGS}; do BAZEL_OPT_FLAGS="${BAZEL_OPT_FLAGS} --copt=${flag}" done; -export CC_OPT_FLAGS - BAZEL_OUTPUT_CACHE_DIR="${DS_ROOT_TASK}/.bazel_cache/" BAZEL_OUTPUT_CACHE_INSTANCE="${BAZEL_OUTPUT_CACHE_DIR}/output/" mkdir -p ${BAZEL_OUTPUT_CACHE_INSTANCE} || true