From 8619665fe1a29df3db3f246573fab2e0a4b5fe09 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Wed, 5 Aug 2020 18:40:57 +0200 Subject: [PATCH] Move to TensorFlow r2.3 --- Dockerfile.build.tmpl | 2 +- native_client/BUILD | 18 +++++++++-- taskcluster/.build.yml | 2 +- taskcluster/.shared.yml | 44 +++++++++++++------------- taskcluster/android-x86_64-cpu-opt.yml | 1 + taskcluster/tc-all-utils.sh | 2 +- taskcluster/tf_tc-vars.sh | 17 ++++++---- tensorflow | 2 +- 8 files changed, 54 insertions(+), 34 deletions(-) diff --git a/Dockerfile.build.tmpl b/Dockerfile.build.tmpl index f64d8e0b..12ce1989 100644 --- a/Dockerfile.build.tmpl +++ b/Dockerfile.build.tmpl @@ -45,7 +45,7 @@ RUN update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 # Install Bazel -RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel_2.0.0-linux-x86_64.deb" +RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel_3.1.0-linux-x86_64.deb" RUN dpkg -i bazel_*.deb # Try and free some space diff --git a/native_client/BUILD b/native_client/BUILD index 0b8ffed3..61bfb19f 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -17,6 +17,20 @@ config_setting( }, ) +config_setting( + name = "rpi3", + define_values = { + "target_system": "rpi3" + }, +) + +config_setting( + name = "rpi3-armv8", + define_values = { + "target_system": "rpi3-armv8" + }, +) + genrule( name = "workspace_status", outs = ["workspace_status.cc"], @@ -132,8 +146,8 @@ tf_cc_shared_object( "//tensorflow:macos": [], "//tensorflow:ios": ["-fembed-bitcode"], "//tensorflow:linux_x86_64": LINUX_LINKOPTS, - "//tensorflow:rpi3": LINUX_LINKOPTS, - "//tensorflow:rpi3-armv8": LINUX_LINKOPTS, + "//native_client:rpi3": LINUX_LINKOPTS, + "//native_client:rpi3-armv8": LINUX_LINKOPTS, "//tensorflow:windows": [], "//conditions:default": [], }) + tflite_linkopts(), diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 2a96f753..13958a13 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -26,7 +26,7 @@ build: nc_asset_name: 'native_client.tar.xz' args: tests_cmdline: '' - tensorflow_git_desc: 'TensorFlow: v2.2.0-24-g1c1b2b9' + tensorflow_git_desc: 'TensorFlow: v2.3.0-4-g4336a5b' test_model_task: '' homebrew: url: '' diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index a25835bc..45f2d982 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -140,38 +140,38 @@ system: namespace: "project.mozilla-voice-stt.swig.win.amd64.1a4c14945012f1282c2eddc174fb7674d5295de8.0" tensorflow: linux_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.cpu/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.cpu" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.cpu/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.cpu" linux_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.cuda/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.cuda/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.cuda" linux_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.arm/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.arm" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.arm/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.arm" linux_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.arm64/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.arm64/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.arm64" darwin_amd64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.osx/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.osx" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.osx/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.osx" android_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.android-arm64/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.android-arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.android-arm64" android_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.android-armv7/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.1.android-armv7" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.android-armv7" win_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.win/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.win" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.win/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.win" win_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.win-cuda/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.win-cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.win-cuda/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.win-cuda" ios_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.ios_arm64/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.ios_arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.ios_arm64/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.ios_arm64" ios_x86_64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.ios_x86_64/artifacts/public/home.tar.xz" - namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.2.1c1b2b9dd9ce767796a8414efbf65da1b67d1622.0.ios_x86_64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.ios_x86_64/artifacts/public/home.tar.xz" + namespace: "project.mozilla-voice-stt.tensorflow.pip.r2.3.4336a5b49fa6d650e24dbdba55bcef9581535244.0.ios_x86_64" username: 'build-user' homedir: linux: '/home/build-user' diff --git a/taskcluster/android-x86_64-cpu-opt.yml b/taskcluster/android-x86_64-cpu-opt.yml index 339dcbf8..f691c4ff 100644 --- a/taskcluster/android-x86_64-cpu-opt.yml +++ b/taskcluster/android-x86_64-cpu-opt.yml @@ -4,6 +4,7 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_android-arm64-opt" routes: - "index.project.mozilla-voice-stt.native_client.${event.head.branchortag}.android-x86_64" - "index.project.mozilla-voice-stt.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64" diff --git a/taskcluster/tc-all-utils.sh b/taskcluster/tc-all-utils.sh index 3f70e526..7e1d940e 100755 --- a/taskcluster/tc-all-utils.sh +++ b/taskcluster/tc-all-utils.sh @@ -101,7 +101,7 @@ verify_bazel_rebuild() cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ - spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libmozilla_voice_stt.so' | wc -l) + spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|[for host]|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libmozilla_voice_stt.so' | wc -l) if [ "${spurious_rebuilds}" -ne 0 ]; then echo "Bazel rebuilds some file it should not, please check." diff --git a/taskcluster/tf_tc-vars.sh b/taskcluster/tf_tc-vars.sh index b768ad4b..0a12372b 100755 --- a/taskcluster/tf_tc-vars.sh +++ b/taskcluster/tf_tc-vars.sh @@ -6,8 +6,8 @@ export OS=$(uname) if [ "${OS}" = "Linux" ]; then export DS_ROOT_TASK=$(/usr/bin/realpath "${HOME}") - BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-linux-x86_64.sh - BAZEL_SHA256=2fbdc9c0e3d376697caf0ee3673b7c9475214068c55a01b9744891e131f90b87 + BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-linux-x86_64.sh + BAZEL_SHA256=7ba815cbac712d061fe728fef958651512ff394b2708e89f79586ec93d1185ed CUDA_URL=http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run CUDA_SHA256=e7c22dc21278eb1b82f34a60ad7640b41ad3943d929bebda3008b72536855d31 @@ -49,8 +49,8 @@ elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then export TEMP=${TASKCLUSTER_TASK_DIR}/tmp/ export TMP=${TASKCLUSTER_TASK_DIR}/tmp/ - BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-windows-x86_64.exe - BAZEL_SHA256=cc7b3ff6f4bfd6bc2121a80656afec66ee57713e8b88e9d2fb58b4eddf271268 + BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-windows-x86_64.exe + BAZEL_SHA256=776db1f4986dacc3eda143932f00f7529f9ee65c7c1c004414c44aaa6419d0e9 CUDA_INSTALL_DIRECTORY=$(cygpath 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1') @@ -68,8 +68,8 @@ elif [ "${OS}" = "Darwin" ]; then export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} - BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-darwin-x86_64.sh - BAZEL_SHA256=c675fa27d99a3114d681db10eb03ded547c40f702b2048c99b8f4ea8e89b9356 + BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-installer-darwin-x86_64.sh + BAZEL_SHA256=5cfa97031b43432b3c742c80e2e01c41c0acdca7ba1052fc8cf1e291271bc9cd SHA_SUM="shasum -a 256 -c" WGET=wget @@ -171,6 +171,11 @@ BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_W BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_CUDA_FLAGS="--config=cuda" +if [ "${OS}" = "Linux" ]; then + # constexpr usage in tensorflow's absl dep fails badly because of gcc-5 + # so let's skip that + BAZEL_CUDA_FLAGS="${BAZEL_CUDA_FLAGS} --copt=-DNO_CONSTEXPR_FOR_YOU=1" +fi BAZEL_IOS_ARM64_FLAGS="--config=ios_arm64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_IOS_X86_64_FLAGS="--config=ios_x86_64 --define=runtime=tflite --copt=-DTFLITE_WITH_RUY_GEMV" diff --git a/tensorflow b/tensorflow index 1c1b2b9d..4336a5b4 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 1c1b2b9dd9ce767796a8414efbf65da1b67d1622 +Subproject commit 4336a5b49fa6d650e24dbdba55bcef9581535244