From 3ce07afae045ce9bf014a36719a0dfec75662187 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 30 Jun 2020 13:01:11 +0200 Subject: [PATCH] Add TensorFlow iOS tasks --- taskcluster/.build.yml | 2 +- taskcluster/.shared.yml | 42 +++++++++++++---------- taskcluster/tf_darwin-amd64-opt.yml | 2 +- taskcluster/tf_ios-arm64-opt.yml | 18 ++++++++++ taskcluster/tf_ios-x86_64-opt.yml | 18 ++++++++++ taskcluster/tf_linux-amd64-cpu-opt.yml | 2 +- taskcluster/tf_tc-build.sh | 46 +++++++++++++------------- taskcluster/tf_tc-vars.sh | 2 ++ taskcluster/tf_win-amd64-cpu-opt.yml | 2 +- tensorflow | 2 +- 10 files changed, 90 insertions(+), 46 deletions(-) create mode 100644 taskcluster/tf_ios-arm64-opt.yml create mode 100644 taskcluster/tf_ios-x86_64-opt.yml diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index fd261359..76fc9703 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -25,7 +25,7 @@ build: nc_asset_name: 'native_client.tar.xz' args: tests_cmdline: '' - tensorflow_git_desc: 'TensorFlow: v2.2.0-15-g518c1d0' + tensorflow_git_desc: 'TensorFlow: v2.2.0-16-gfc46411' test_model_task: '' homebrew: url: '' diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 7ec3f366..9323c995 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -142,32 +142,38 @@ system: namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" tensorflow: linux_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.cpu/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.cpu" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.cpu/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.cpu" linux_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.cuda/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.cuda" linux_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.arm/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.arm" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.arm/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.arm" linux_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.arm64" darwin_amd64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.3.osx/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.3.osx" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.osx/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.osx" android_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.android-arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.android-arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.android-arm64" android_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.android-armv7/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.android-armv7" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.android-armv7" win_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.win/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.win" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.win/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.win" win_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.win-cuda/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.0854bb5188a3150a4d75a1c71ee610b0d45cfcb1.win-cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.win-cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.win-cuda" + ios_arm64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.ios_arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.ios_arm64" + ios_x86_64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.ios_x86_64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.fc464111ac5c49791e44122e5946e521b25840bd.ios_x86_64" username: 'build-user' homedir: linux: '/home/build-user' diff --git a/taskcluster/tf_darwin-amd64-opt.yml b/taskcluster/tf_darwin-amd64-opt.yml index 21c19021..365e1700 100644 --- a/taskcluster/tf_darwin-amd64-opt.yml +++ b/taskcluster/tf_darwin-amd64-opt.yml @@ -10,7 +10,7 @@ build: ${tensorflow.packages_macos.brew} scripts: setup: "taskcluster/tf_tc-setup.sh" - build: "taskcluster/tf_tc-build.sh --osx" + build: "taskcluster/tf_tc-build.sh --cpu" package: "taskcluster/tf_tc-package.sh" maxRunTime: 28800 metadata: diff --git a/taskcluster/tf_ios-arm64-opt.yml b/taskcluster/tf_ios-arm64-opt.yml new file mode 100644 index 00000000..9f253b3f --- /dev/null +++ b/taskcluster/tf_ios-arm64-opt.yml @@ -0,0 +1,18 @@ +build: + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.ios_arm64.url} + artifact_namespace: ${system.tensorflow.ios_arm64.namespace} + generic: + workerType: "ds-macos-heavy" + system_config: + > + ${tensorflow.packages_macos.brew} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --ios-arm64" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 28800 + metadata: + name: "TensorFlow iOS ARM64 TFLite" + description: "Building TensorFlow for iOS ARM64, TFLite, optimized version" diff --git a/taskcluster/tf_ios-x86_64-opt.yml b/taskcluster/tf_ios-x86_64-opt.yml new file mode 100644 index 00000000..c56ad3ca --- /dev/null +++ b/taskcluster/tf_ios-x86_64-opt.yml @@ -0,0 +1,18 @@ +build: + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.ios_x86_64.url} + artifact_namespace: ${system.tensorflow.ios_x86_64.namespace} + generic: + workerType: "ds-macos-heavy" + system_config: + > + ${tensorflow.packages_macos.brew} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --ios-x86_64" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 28800 + metadata: + name: "TensorFlow iOS x86_64 TFLite" + description: "Building TensorFlow for iOS x86_64, TFLite, optimized version" diff --git a/taskcluster/tf_linux-amd64-cpu-opt.yml b/taskcluster/tf_linux-amd64-cpu-opt.yml index d869d280..36b051b1 100644 --- a/taskcluster/tf_linux-amd64-cpu-opt.yml +++ b/taskcluster/tf_linux-amd64-cpu-opt.yml @@ -8,7 +8,7 @@ build: ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} scripts: setup: "taskcluster/tf_tc-setup.sh" - build: "taskcluster/tf_tc-build.sh" + build: "taskcluster/tf_tc-build.sh --cpu" package: "taskcluster/tf_tc-package.sh" maxRunTime: 14400 metadata: diff --git a/taskcluster/tf_tc-build.sh b/taskcluster/tf_tc-build.sh index ad8085ee..cb620906 100755 --- a/taskcluster/tf_tc-build.sh +++ b/taskcluster/tf_tc-build.sh @@ -4,56 +4,48 @@ set -ex source $(dirname $0)/tf_tc-vars.sh -build_amd64=yes +build_amd64=no build_gpu=no build_android_arm=no build_android_arm64=no build_linux_arm=no build_linux_arm64=no +build_ios_arm64=no +build_ios_x86_64=no + +if [ "$1" = "--cpu" ]; then + build_amd64=yes +fi if [ "$1" = "--gpu" ]; then build_amd64=yes build_gpu=yes - build_android_arm=no - build_android_arm64=no - build_linux_arm=no - build_linux_arm64=no fi if [ "$1" = "--arm" ]; then build_amd64=yes - build_gpu=no - build_android_arm=no - build_android_arm64=no build_linux_arm=yes - build_linux_arm64=no fi if [ "$1" = "--arm64" ]; then build_amd64=yes - build_gpu=no - build_android_arm=no - build_android_arm64=no - build_linux_arm=no build_linux_arm64=yes fi if [ "$1" = "--android-armv7" ]; then - build_amd64=no - build_gpu=no build_android_arm=yes - build_android_arm64=no - build_linux_arm=no - build_linux_arm64=no fi if [ "$1" = "--android-arm64" ]; then - build_amd64=no - build_gpu=no - build_android_arm=no build_android_arm64=yes - build_linux_arm=no - build_linux_arm64=no +fi + +if [ "$1" = "--ios-arm64" ]; then + build_ios_arm64=yes +fi + +if [ "$1" = "--ios-x86_64" ]; then + build_ios_x86_64=yes fi pushd ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/ @@ -98,6 +90,14 @@ pushd ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/ echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} fi; + if [ "${build_ios_arm64}" = "yes" ]; then + echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_IOS_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + fi; + + if [ "${build_ios_x86_64}" = "yes" ]; then + echo "" | TF_NEED_CUDA=0 TF_CONFIGURE_IOS=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_IOS_X86_64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + fi; + if [ $? -ne 0 ]; then # There was a failure, just account for it. echo "Build failure, please check the output above. Exit code was: $?" diff --git a/taskcluster/tf_tc-vars.sh b/taskcluster/tf_tc-vars.sh index 8150bb8d..8c30ea2a 100755 --- a/taskcluster/tf_tc-vars.sh +++ b/taskcluster/tf_tc-vars.sh @@ -171,6 +171,8 @@ BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_W BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_CUDA_FLAGS="--config=cuda" +BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64 --copt=-DTFLITE_WITH_RUY_GEMV" +BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64 --copt=-DTFLITE_WITH_RUY_GEMV" if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then # Somehow, even with Python being in the PATH, Bazel on windows struggles diff --git a/taskcluster/tf_win-amd64-cpu-opt.yml b/taskcluster/tf_win-amd64-cpu-opt.yml index 4b4d1105..99b4d8a3 100644 --- a/taskcluster/tf_win-amd64-cpu-opt.yml +++ b/taskcluster/tf_win-amd64-cpu-opt.yml @@ -8,7 +8,7 @@ build: ${tensorflow.packages_win.pacman} && ${tensorflow.packages_win.msys64} scripts: setup: "taskcluster/tf_tc-setup.sh" - build: "taskcluster/tf_tc-build.sh" + build: "taskcluster/tf_tc-build.sh --cpu" package: "taskcluster/tf_tc-package.sh" maxRunTime: 14400 metadata: diff --git a/tensorflow b/tensorflow index 518c1d04..fc464111 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d +Subproject commit fc464111ac5c49791e44122e5946e521b25840bd