Use TensorFlow as a submodule

.gitmodules

@@ -2,3 +2,6 @@
 	path = doc/examples
 	url = https://github.com/mozilla/DeepSpeech-examples.git
 	branch = master
+[submodule "tensorflow"]
+	path = tensorflow
+	url = https://github.com/mozilla/tensorflow.git

Dockerfile.build.tmpl

@@ -53,11 +53,6 @@ RUN dpkg -i bazel_*.deb
 
 # >> START Configure Tensorflow Build
 
-# Clone TensorFlow from Mozilla repo
-RUN git clone https://github.com/mozilla/tensorflow/
-WORKDIR /tensorflow
-RUN git checkout r2.2
-
 # GPU Environment Setup
 ENV TF_NEED_ROCM 0
 ENV TF_NEED_OPENCL_SYCL 0
@@ -116,16 +111,15 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
 
 WORKDIR /
 
-RUN git clone $DEEPSPEECH_REPO
+RUN git clone --recursive $DEEPSPEECH_REPO
 WORKDIR /DeepSpeech
 RUN git checkout $DEEPSPEECH_SHA
-
+RUN git submodule sync tensorflow/
-# Link DeepSpeech native_client libs to tf folder
+RUN git submodule update --init tensorflow/
-RUN ln -s /DeepSpeech/native_client /tensorflow
 
 # >> START Build and bind
 
-WORKDIR /tensorflow
+WORKDIR /DeepSpeech/tensorflow
 
 # Fix for not found script https://github.com/tensorflow/tensorflow/issues/471
 RUN ./configure
@@ -158,10 +152,10 @@ RUN bazel build \
 	--action_env=LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
 
 # Copy built libs to /DeepSpeech/native_client
-RUN cp /tensorflow/bazel-bin/native_client/libdeepspeech.so /DeepSpeech/native_client/
+RUN cp bazel-bin/native_client/libdeepspeech.so /DeepSpeech/native_client/
 
 # Build client.cc and install Python client and decoder bindings
-ENV TFDIR /tensorflow
+ENV TFDIR /DeepSpeech/tensorflow
 
 RUN nproc
 

native_client/README.rst

@@ -4,10 +4,8 @@ Building DeepSpeech Binaries
 
 If you'd like to build the DeepSpeech binaries yourself, you'll need the following pre-requisites downloaded and installed:
 
-
-* `Mozilla's TensorFlow r2.2 branch <https://github.com/mozilla/tensorflow/tree/r2.2>`_
 * `Bazel 2.0.0 <https://github.com/bazelbuild/bazel/releases/tag/2.0.0>`_
-* `General TensorFlow requirements <https://www.tensorflow.org/install/install_sources>`_
+* `General TensorFlow r2.2 requirements <https://www.tensorflow.org/install/source#tested_build_configurations>`_
 * `libsox <https://sourceforge.net/projects/sox/>`_
 
 It is required to use our fork of TensorFlow since it includes fixes for common problems encountered when building the native client files.
@@ -28,15 +26,16 @@ If you follow these instructions, you should compile your own binaries of DeepSp
 
 For more information on configuring TensorFlow, read the docs up to the end of `"Configure the Build" <https://www.tensorflow.org/install/source#configure_the_build>`_.
 
-TensorFlow: Clone & Checkout
+Checkout source code
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^
 
-Clone our fork of TensorFlow and checkout the correct version:
+Clone DeepSpeech source code (TensorFlow will come as a submdule):
 
 .. code-block::
 
-   git clone https://github.com/mozilla/tensorflow.git
+   git clone https://github.com/mozilla/DeepSpeech.git
-   git checkout origin/r2.2
+   git submodule sync tensorflow/
+   git submodule update --init tensorflow/
 
 Bazel: Download & Install
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -57,16 +56,16 @@ Compile DeepSpeech
 ------------------
 
 Compile ``libdeepspeech.so``
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Within your TensorFlow checkout, create a symbolic link to the DeepSpeech ``native_client`` directory. Assuming DeepSpeech and TensorFlow checkouts are in the same directory, do:
+Within your TensorFlow directory, there should be a symbolic link to the DeepSpeech ``native_client`` directory. If it is not present, create it with the follow command:
 
 .. code-block::
 
    cd tensorflow
-   ln -s ../DeepSpeech/native_client ./
+   ln -s ../native_client
 
-You can now use Bazel to build the main DeepSpeech library, ``libdeepspeech.so``\ . Add ``--config=cuda`` if you want a CUDA build.
+You can now use Bazel to build the main DeepSpeech library, ``libdeepspeech.so``. Add ``--config=cuda`` if you want a CUDA build.
 
 .. code-block::
 
@@ -77,11 +76,10 @@ The generated binaries will be saved to ``bazel-bin/native_client/``.
 Compile Language Bindings
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Now, ``cd`` into the ``DeepSpeech/native_client`` directory and use the ``Makefile`` to build all the language bindings (C++ client, Python package, Nodejs package, etc.). Set the environment variable ``TFDIR`` to point to your TensorFlow checkout.
+Now, ``cd`` into the ``DeepSpeech/native_client`` directory and use the ``Makefile`` to build all the language bindings (C++ client, Python package, Nodejs package, etc.).
 
 .. code-block::
 
-   TFDIR=~/tensorflow
    cd ../DeepSpeech/native_client
    make deepspeech
 
@@ -191,11 +189,11 @@ Building the ``deepspeech`` binary will happen through ``ndk-build`` (ARMv7):
 .. code-block::
 
    cd ../DeepSpeech/native_client
-   $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../../tensorflow/ TARGET_ARCH_ABI=armeabi-v7a
+   $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../tensorflow/ TARGET_ARCH_ABI=armeabi-v7a
 
 And (ARM64):
 
 .. code-block::
 
    cd ../DeepSpeech/native_client
-   $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../../tensorflowx/ TARGET_ARCH_ABI=arm64-v8a
+   $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../tensorflow/ TARGET_ARCH_ABI=arm64-v8a

native_client/definitions.mk

@@ -1,7 +1,7 @@
 NC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 
 TARGET    ?= host
-TFDIR     ?= $(abspath $(NC_DIR)/../../tensorflow)
+TFDIR     ?= $(abspath $(NC_DIR)/../tensorflow)
 PREFIX    ?= /usr/local
 SO_SEARCH ?= $(TFDIR)/bazel-bin/
 

native_client/dotnet/README.rst

@@ -43,36 +43,34 @@ We highly recommend sticking to the recommended versions of CUDA/cuDNN in order
 Getting the code
 ----------------
 
-We need to clone ``mozilla/DeepSpeech`` and ``mozilla/tensorflow``.
+We need to clone ``mozilla/DeepSpeech``.
 
 .. code-block:: bash
 
    git clone https://github.com/mozilla/DeepSpeech
-
+   git submodule sync tensorflow/
-.. code-block:: bash
+   git submodule update --init tensorflow/
-
-   git clone --branch r2.2 https://github.com/mozilla/tensorflow
 
 Configuring the paths
 ---------------------
 
-We need to create a symbolic link, for this example let's suppose that we cloned into ``D:\cloned`` and now the structure looks like:
+There should already be a symbolic link, for this example let's suppose that we cloned into ``D:\cloned`` and now the structure looks like:
 
 .. code-block::
 
    .
    ├── D:\
    │   ├── cloned                 # Contains DeepSpeech and tensorflow side by side
-   │   │   ├── DeepSpeech         # Root of the cloned DeepSpeech
+   │   │   └── DeepSpeech         # Root of the cloned DeepSpeech
-   │   │   ├── tensorflow         # Root of the cloned Mozilla's tensorflow 
+   │   │       ├── tensorflow     # Root of the cloned Mozilla's tensorflow 
    └── ...
 
 
-Change your path accordingly to your path structure, for the structure above we are going to use the following command:
+Change your path accordingly to your path structure, for the structure above we are going to use the following command if the symbolic link does not exists:
 
 .. code-block:: bash
 
-   mklink /d "D:\cloned\tensorflow\native_client" "D:\cloned\DeepSpeech\native_client"
+   mklink /d "D:\cloned\DeepSpeech\tensorflow\native_client" "D:\cloned\DeepSpeech\native_client"
 
 Adding environment variables
 ----------------------------
@@ -82,7 +80,7 @@ After you have installed the requirements there are few environment variables th
 MSYS2 paths
 ~~~~~~~~~~~
 
-For MSYS2 we need to add ``bin`` directory, if you installed in the default route the path that we need to add should looks like ``C:\msys64\usr\bin``. Now we can run ``pacman``\ :
+For MSYS2 we need to add ``bin`` directory, if you installed in the default route the path that we need to add should looks like ``C:\msys64\usr\bin``. Now we can run ``pacman``:
 
 .. code-block:: bash
 
@@ -120,7 +118,7 @@ Building the native_client
 
 There's one last command to run before building, you need to run the `configure.py <https://github.com/mozilla/tensorflow/blob/master/configure.py>`_ inside ``tensorflow`` cloned directory.
 
-At this point we are ready to start building the ``native_client``\ , go to ``tensorflow`` directory that you cloned, following our examples should be ``D:\cloned\tensorflow``.  
+At this point we are ready to start building the ``native_client``, go to ``tensorflow`` sub-directory, following our examples should be ``D:\cloned\DeepSpeech\tensorflow``.  
 
 CPU
 ~~~

taskcluster/.build.yml

@@ -10,6 +10,8 @@ build:
   routes: []
   maxRunTime: 3600
   docker_image: "ubuntu:16.04"
+  generic:
+    workerType: 'ds-macos-light'
   system_setup:
     >
       true
@@ -17,12 +19,13 @@ build:
     >
       true
   scripts:
+    setup: ''
     build: ''
     package: ''
   nc_asset_name: 'native_client.tar.xz'
   args:
     tests_cmdline: ''
-  tensorflow_git_desc: 'TensorFlow: v2.2.0-12-gc29895f'
+  tensorflow_git_desc: 'TensorFlow: v2.2.0-14-g7ead558'
   test_model_task: ''
   homebrew:
     url: ''

taskcluster/.shared.yml

@@ -8,9 +8,17 @@ python:
 training:
   packages_xenial:
     apt: 'libopus0'
+deepspeech:
+  packages_xenial:
+    apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev'
 tensorflow:
   packages_xenial:
-    apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python zlib1g-dev'
+    apt: 'apt-get -qq update && apt-get -qq -y install realpath build-essential python-virtualenv python-dev python-pip libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip'
+  packages_macos:
+    brew: '$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tf_tc-brew.sh'
+  packages_win:
+    pacman: 'pacman --noconfirm -S patch unzip tar'
+    msys64: 'ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64'
 java:
   packages_xenial:
     apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f'
@@ -132,6 +140,34 @@ system:
     win:
       url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz"
       namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
+  tensorflow:
+    linux_amd64_cpu:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cpu/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cpu"
+    linux_amd64_cuda:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cuda/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cuda"
+    linux_armv7:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm"
+    linux_arm64:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm64/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm64"
+    darwin_amd64:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.osx/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.osx"
+    android_arm64:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-arm64/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-arm64"
+    android_armv7:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-armv7/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-armv7"
+    win_amd64_cpu:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win"
+    win_amd64_cuda:
+      url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win-cuda/artifacts/public/home.tar.xz"
+      namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win-cuda"
   username: 'build-user'
   homedir:
     linux: '/home/build-user'
@@ -141,7 +177,3 @@ system:
   msys2:
     url: 'https://github.com/msys2/msys2-installer/releases/download/2020-06-02/msys2-base-x86_64-20200602.tar.xz'
     sha: '598ceeaa3e2ccf86a25a2e3c449d00a9fd35300e36011bee610036dfa59d670a'
-  msys2_filesystem_pkg:
-    url: 'http://repo.msys2.org/msys/x86_64/filesystem-2020.02-3-x86_64.pkg.tar.xz'
-    sha: '927b020a67a05139ee1b2c45bff491c1d42335e64350cc7758ee20d7c3099477'
-    install: 'pacman -Udd --noconfirm $USERPROFILE/filesystem-2020.02-3-x86_64.pkg.tar.xz'

taskcluster/android-apk-build.sh

@@ -6,6 +6,6 @@ arm_flavor=$1
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 do_deepspeech_java_apk_build

taskcluster/android-apk-package.sh

@@ -8,7 +8,7 @@ source $(dirname "$0")/tc-tests-utils.sh
 
 mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
 
-cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
+cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
 
 cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/java/app/build/outputs/apk/release/app*.apk ${TASKCLUSTER_ARTIFACTS}/
 cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/java/libdeepspeech/build/outputs/aar/libdeepspeech*.aar ${TASKCLUSTER_ARTIFACTS}/

taskcluster/android-arm64-cpu-opt.yml

@@ -4,12 +4,14 @@ build:
     - "swig-linux-amd64"
     - "node-gyp-cache"
     - "pyenv-linux-amd64"
+    - "tf_android-arm64-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64"
     - "index.project.deepspeech.deepspeech.native_client.android-arm64.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-arm64/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.android_arm64.url}
   scripts:
+    setup: ""
     build: "taskcluster/android-build.sh arm64-v8a"
     package: "taskcluster/android-package.sh arm64-v8a"
   nc_asset_name: "native_client.arm64.cpu.android.tar.xz"

taskcluster/android-armv7-cpu-opt.yml

@@ -4,11 +4,12 @@ build:
     - "swig-linux-amd64"
     - "node-gyp-cache"
     - "pyenv-linux-amd64"
+    - "tf_android-armv7-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7"
     - "index.project.deepspeech.deepspeech.native_client.android-armv7.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-armv7/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.android_armv7.url}
   scripts:
     build: "taskcluster/android-build.sh armeabi-v7a"
     package: "taskcluster/android-package.sh armeabi-v7a"

taskcluster/android-build.sh

@@ -6,7 +6,7 @@ arm_flavor=$1
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 BAZEL_TARGETS="
 //native_client:libdeepspeech.so

taskcluster/android-cache-arm64-v8a-android-24.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.arm64_v8a.android_24.url}
     artifact_namespace: ${system.android_cache.arm64_v8a.android_24.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh arm64-v8a android-24"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-arm64-v8a-android-25.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.arm64_v8a.android_25.url}
     artifact_namespace: ${system.android_cache.arm64_v8a.android_25.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh arm64-v8a android-25"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-armeabi-v7a-android-24.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.armeabi_v7a.android_24.url}
     artifact_namespace: ${system.android_cache.armeabi_v7a.android_24.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh armeabi-v7a android-24 default"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-armeabi-v7a-android-25.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.armeabi_v7a.android_25.url}
     artifact_namespace: ${system.android_cache.armeabi_v7a.android_25.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh armeabi-v7a android-25"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-sdk-android-27.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.sdk.android_27.url}
     artifact_namespace: ${system.android_cache.sdk.android_27.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh sdk android-27"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-x86_64-android-24.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.x86_64.android_24.url}
     artifact_namespace: ${system.android_cache.x86_64.android_24.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh x86_64 android-24"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-x86_64-android-25.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.x86_64.android_25.url}
     artifact_namespace: ${system.android_cache.x86_64.android_25.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh x86_64 android-25"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-x86_64-android-26.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.x86_64.android_26.url}
     artifact_namespace: ${system.android_cache.x86_64.android_26.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh x86_64 android-26"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-x86_64-android-28.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.x86_64.android_28.url}
     artifact_namespace: ${system.android_cache.x86_64.android_28.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh x86_64 android-28"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-x86_64-android-29.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.x86_64.android_29.url}
     artifact_namespace: ${system.android_cache.x86_64.android_29.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh x86_64 android-29"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-cache-x86_64-android-30.yml

@@ -7,6 +7,7 @@ build:
     artifact_url: ${system.android_cache.x86_64.android_30.url}
     artifact_namespace: ${system.android_cache.x86_64.android_30.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/android_cache-build.sh x86_64 android-30"
     package: "taskcluster/android_cache-package.sh"
   metadata:

taskcluster/android-java-opt.yml

@@ -14,7 +14,7 @@ build:
   system_setup:
     >
       ${java.packages_xenial.apt}
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-armv7/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.android_armv7.url}
   gradle_cache:
     url: ${system.gradle_cache.url}
     namespace: ${system.gradle_cache.namespace}

taskcluster/android-package.sh

@@ -8,6 +8,6 @@ source $(dirname "$0")/tc-tests-utils.sh
 
 mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
 
-cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
+cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
 
 package_native_client_ndk "native_client.tar.xz" "${arm_flavor}"

taskcluster/android-x86_64-cpu-opt.yml

@@ -8,7 +8,7 @@ build:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64"
     - "index.project.deepspeech.deepspeech.native_client.android-x86_64.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-arm64/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.android_arm64.url}
   scripts:
     build: "taskcluster/android-build.sh x86_64"
     package: "taskcluster/android-package.sh x86_64"

taskcluster/arm64-build.sh

@@ -4,7 +4,7 @@ set -xe
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 BAZEL_TARGETS="
 //native_client:libdeepspeech.so

taskcluster/cuda-build.sh

@@ -4,7 +4,7 @@ set -xe
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 BAZEL_TARGETS="
 //native_client:libdeepspeech.so

taskcluster/darwin-amd64-cpu-opt.yml

@@ -5,11 +5,12 @@ build:
     - "node-gyp-cache"
     - "homebrew_builds-darwin-amd64"
     - "pyenv-darwin-amd64"
+    - "tf_darwin-amd64-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx"
     - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.darwin_amd64.url}
   scripts:
     build: "taskcluster/host-build.sh"
     package: "taskcluster/package.sh"

taskcluster/darwin-amd64-ctc-opt.yml

@@ -5,11 +5,12 @@ build:
     - "node-gyp-cache"
     - "homebrew_builds-darwin-amd64"
     - "pyenv-darwin-amd64"
+    - "tf_darwin-amd64-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc"
     - "index.project.deepspeech.deepspeech.native_client.osx-ctc.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.darwin_amd64.url}
   maxRunTime: 14400
   scripts:
     build: 'taskcluster/decoder-build.sh'

taskcluster/darwin-amd64-tflite-opt.yml

@@ -5,11 +5,12 @@ build:
     - "node-gyp-cache"
     - "homebrew_builds-darwin-amd64"
     - "pyenv-darwin-amd64"
+    - "tf_darwin-amd64-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite"
     - "index.project.deepspeech.deepspeech.native_client.osx-tflite.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.darwin_amd64.url}
   scripts:
     build: "taskcluster/host-build.sh tflite"
     package: "taskcluster/package.sh"

taskcluster/darwin-opt-base.tyml

@@ -60,9 +60,8 @@ payload:
         cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ &&
         exit $TASKCLUSTER_TASK_EXIT_CODE" 0 &&
         (pixz -d < $TASKCLUSTER_ORIG_TASKDIR/home.tar.xz | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
-        git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
+        cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} &&
-        cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
+        git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ &&
-        ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
         cd $TASKCLUSTER_TASK_DIR &&
         (mkdir pyenv-root/ && gtar -C pyenv-root/ -xf $TASKCLUSTER_ORIG_TASKDIR/pyenv.tar.gz) &&
         (mkdir homebrew-builds/ && gtar -C homebrew-builds/ -xf $TASKCLUSTER_ORIG_TASKDIR/homebrew-builds.tar.gz) &&

taskcluster/decoder-build.sh

@@ -4,7 +4,7 @@ set -xe
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
   export SYSTEM_TARGET=host-win

taskcluster/generic_tc_caching-darwin-opt-base.tyml

@@ -1,6 +1,6 @@
 taskId: ${taskcluster.taskId}
 provisionerId: ${taskcluster.generic.provisionerId}
-workerType: ${taskcluster.generic.workerType}
+workerType: ${build.generic.workerType}
 taskGroupId: ${taskcluster.taskGroupId}
 schedulerId: ${taskcluster.schedulerId}
 dependencies:
@@ -24,6 +24,8 @@ payload:
       - "--login"
       - "-cxe"
       - $let:
+          extraSystemSetup: { $eval: strip(str(build.system_setup)) }
+          extraSystemConfig: { $eval: strip(str(build.system_config)) }
           taskIndexExpire: { $fromNow: '6 months' }
         in: >
           export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
@@ -32,13 +34,14 @@ payload:
           export TASKCLUSTER_TASK_DIR="$(pwd)" &&
           export LC_ALL=C &&
           export MACOSX_DEPLOYMENT_TARGET=10.10 &&
-          export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
           env &&
           mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
           cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` &&
           if [ "$cache_artifact" != "200" ]; then
             git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} &&
             cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} &&
+            ${extraSystemConfig} &&
+            $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.setup} &&
             $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} &&
             $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} &&
             $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} 127.0.0.1:8080 ${build.cache.artifact_namespace}

taskcluster/generic_tc_caching-linux-opt-base.tyml

@@ -22,6 +22,7 @@ payload:
     - "-cxe"
     - $let:
         extraSystemSetup: { $eval: strip(str(build.system_setup)) }
+        extraSystemConfig: { $eval: strip(str(build.system_config)) }
         taskIndexExpire: { $fromNow: '6 months' }
       in: >
         (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
@@ -31,7 +32,8 @@ payload:
           adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ &&
           mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts &&
           echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${build.build_or_cache.repo} ~/${build.build_or_cache.dir}/ && cd ~/${build.build_or_cache.dir}/ && git checkout --quiet ${build.build_or_cache.sha})" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
-          sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
+          sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
+          sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.setup} &&
           sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.build} &&
           sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.package} &&
           sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}

taskcluster/generic_tc_caching-win-opt-base.tyml

@@ -20,10 +20,6 @@ payload:
       content:
         sha256: ${system.msys2.sha}
         url: ${system.msys2.url}
-    - file: filesystem-2020.02-3-x86_64.pkg.tar.xz
-      content:
-        sha256: ${system.msys2_filesystem_pkg.sha}
-        url: ${system.msys2_filesystem_pkg.url}
 
   env:
     TC_MSYS_VERSION: 'MSYS_NT-6.3-9600'
@@ -34,18 +30,19 @@ payload:
       "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
       "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
     - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
-    - .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}"
-    - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
     - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
     - $let:
+        extraSystemSetup: { $eval: strip(str(build.system_setup)) }
+        extraSystemConfig: { $eval: strip(str(build.system_config)) }
         taskIndexExpire: { $fromNow: '6 months' }
       in: >
         echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C &&
         export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" &&
         export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" &&
         export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
+        (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) &&
         echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh &&
-        env && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k
+        env && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && ${extraSystemConfig} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.setup} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k
 
     - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE"
 

taskcluster/gradle-cache.yml

@@ -7,6 +7,7 @@ build:
     >
       ${java.packages_xenial.apt}
   scripts:
+    setup: ""
     build: "taskcluster/gradle-build.sh"
     package: "taskcluster/gradle-package.sh"
   metadata:

taskcluster/homebrew_builds-darwin-amd64.yml

@@ -4,6 +4,7 @@ build:
     artifact_url: ${system.homebrew_builds.url}
     artifact_namespace: ${system.homebrew_builds.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/homebrew-build.sh --builds"
     package: "taskcluster/homebrew-package.sh --builds"
   metadata:

taskcluster/homebrew_tests-darwin-amd64.yml

@@ -4,6 +4,7 @@ build:
     artifact_url: ${system.homebrew_tests.url}
     artifact_namespace: ${system.homebrew_tests.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/homebrew-build.sh --tests"
     package: "taskcluster/homebrew-package.sh --tests"
   metadata:

taskcluster/host-build.sh

@@ -6,7 +6,7 @@ runtime=$1
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 BAZEL_TARGETS="
 //native_client:libdeepspeech.so

taskcluster/linux-amd64-cpu-opt.yml

@@ -4,6 +4,7 @@ build:
     - "swig-linux-amd64"
     - "node-gyp-cache"
     - "pyenv-linux-amd64"
+    - "tf_linux-amd64-cpu-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu"
@@ -12,8 +13,9 @@ build:
     >
       ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning}
       && apt-get -qq update && apt-get -qq -y install nodejs python-yaml
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.linux_amd64_cpu.url}
   scripts:
+    setup: ""
     build: "taskcluster/host-build.sh"
     package: "taskcluster/package.sh"
   nc_asset_name: "native_client.amd64.cpu.linux.tar.xz"

taskcluster/linux-amd64-ctc-opt.yml

@@ -4,6 +4,7 @@ build:
     - "swig-linux-amd64"
     - "node-gyp-cache"
     - "pyenv-linux-amd64"
+    - "tf_linux-amd64-cpu-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-ctc"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-ctc"
@@ -12,8 +13,9 @@ build:
     >
       ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning}
       && apt-get -qq update && apt-get -qq -y install nodejs python-yaml
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.linux_amd64_cpu.url}
   scripts:
+    setup: ""
     build: 'taskcluster/decoder-build.sh'
     package: 'taskcluster/decoder-package.sh'
   metadata:

taskcluster/linux-amd64-gpu-opt.yml

@@ -4,6 +4,7 @@ build:
     - "swig-linux-amd64"
     - "node-gyp-cache"
     - "pyenv-linux-amd64"
+    - "tf_linux-amd64-gpu-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu"
@@ -12,9 +13,10 @@ build:
     >
       ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning}
       && apt-get -qq update && apt-get -qq -y install nodejs python-yaml
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.gpu/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.linux_amd64_cuda.url}
   maxRunTime: 14400
   scripts:
+    setup: ""
     build: "taskcluster/cuda-build.sh"
     package: "taskcluster/package.sh"
   nc_asset_name: "native_client.amd64.cuda.linux.tar.xz"

taskcluster/linux-amd64-tflite-opt.yml

@@ -4,6 +4,7 @@ build:
     - "swig-linux-amd64"
     - "node-gyp-cache"
     - "pyenv-linux-amd64"
+    - "tf_linux-amd64-cpu-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite"
@@ -12,8 +13,9 @@ build:
     >
       ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning}
       && apt-get -qq update && apt-get -qq -y install nodejs python-yaml
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.linux_amd64_cpu.url}
   scripts:
+    setup: ""
     build: "taskcluster/host-build.sh tflite"
     package: "taskcluster/package.sh"
   nc_asset_name: "native_client.amd64.tflite.linux.tar.xz"

taskcluster/linux-arm64-cpu-opt.yml

@@ -4,6 +4,7 @@ build:
     - "swig-linux-amd64"
     - "node-gyp-cache"
     - "pyenv-linux-amd64"
+    - "tf_linux-arm64-cpu-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
@@ -19,8 +20,9 @@ build:
   system_config:
     >
       multistrap -d /tmp/multistrap-armbian64-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_armbian64_buster.conf
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.arm64/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.linux_arm64.url}
   scripts:
+    setup: ""
     build: "taskcluster/arm64-build.sh"
     package: "taskcluster/package.sh"
   nc_asset_name: "native_client.arm64.cpu.linux.tar.xz"

taskcluster/linux-opt-base.tyml

@@ -44,9 +44,9 @@ then:
           extraSystemConfig: { $eval: strip(str(build.system_config)) }
         in: >
           adduser --system --home ${system.homedir.linux} ${system.username} &&
-          apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} &&
+          apt-get -qq update && apt-get -qq -y install ${deepspeech.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} &&
           cd ${system.homedir.linux}/ &&
-          echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && cd ~/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
           sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
           sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
           sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package}

taskcluster/linux-rpi3-cpu-opt.yml

@@ -4,6 +4,7 @@ build:
     - "swig-linux-amd64"
     - "node-gyp-cache"
     - "pyenv-linux-amd64"
+    - "tf_linux-rpi3-cpu-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
@@ -19,8 +20,9 @@ build:
   system_config:
     >
       multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.arm/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.linux_armv7.url}
   scripts:
+    setup: ""
     build: "taskcluster/rpi3-build.sh"
     package: "taskcluster/package.sh"
   nc_asset_name: "native_client.rpi3.cpu.linux.tar.xz"

taskcluster/node-build.sh

@@ -6,6 +6,6 @@ package_option=$1
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 do_deepspeech_npm_package "${package_option}"

taskcluster/node-gyp-cache.yml

@@ -8,6 +8,7 @@ build:
     >
       (apt-get -qq -y install sudo || true)
   scripts:
+    setup: ""
     build: "taskcluster/node-gyp-populate.sh"
     package: "taskcluster/node-gyp-package.sh"
   metadata:

taskcluster/node-package-opt-base.tyml

@@ -43,7 +43,7 @@ then:
           adduser --system --home ${system.homedir.linux} ${system.username} &&
           apt-get -qq update && apt-get -qq -y install realpath git wget curl make sudo && ${extraSystemSetup} &&
           cd ${system.homedir.linux}/ &&
-          echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ~/DeepSpeech/tf/ && touch ~/DeepSpeech/tf/tc-vars.sh && chmod +x ~/DeepSpeech/tf/tc-vars.sh && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
+          echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
           sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
           sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
           sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package}

taskcluster/package.sh

@@ -6,7 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh
 
 mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
 
-cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
+cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
 
 package_native_client "native_client.tar.xz"
 

taskcluster/pyenv-darwin-amd64.yml

@@ -4,6 +4,7 @@ build:
     artifact_url: ${system.pyenv.osx.url}
     artifact_namespace: ${system.pyenv.osx.namespace}
   scripts:
+    setup: ""
     build: "taskcluster/pyenv-build.sh"
     package: "taskcluster/pyenv-package.sh"
   metadata:

taskcluster/pyenv-linux-amd64.yml

@@ -7,6 +7,7 @@ build:
     >
       apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_xenial.apt} wget
   scripts:
+    setup: ""
     build: "taskcluster/pyenv-build.sh"
     package: "taskcluster/pyenv-package.sh"
   metadata:

taskcluster/pyenv-win-amd64.yml

@@ -4,6 +4,7 @@ build:
     artifact_url: "${system.pyenv.win.url}"
     artifact_namespace: "${system.pyenv.win.namespace}"
   scripts:
+    setup: ""
     build: "taskcluster/pyenv-build.sh"
     package: "taskcluster/pyenv-package.sh"
   metadata:

taskcluster/rpi3-build.sh

@@ -4,7 +4,7 @@ set -xe
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 BAZEL_TARGETS="
 //native_client:libdeepspeech.so

taskcluster/swig-darwin-amd64.yml

@@ -8,6 +8,7 @@ build:
     artifact_url: "${system.swig_build.osx.url}"
     artifact_namespace: "${system.swig_build.osx.namespace}"
   scripts:
+    setup: ""
     build: "taskcluster/build.sh"
     package: "taskcluster/package.sh"
   metadata:

taskcluster/swig-linux-amd64.yml

@@ -12,6 +12,7 @@ build:
     >
       apt-get -qq -y install autoconf automake bison build-essential
   scripts:
+    setup: ""
     build: "taskcluster/build.sh"
     package: "taskcluster/package.sh"
   metadata:

taskcluster/swig-win-amd64.yml

@@ -13,6 +13,7 @@ build:
       apt-get -qq -y install autoconf automake bison build-essential mingw-w64 &&
       (apt-get -qq -y install sudo || true)
   scripts:
+    setup: ""
     build: "taskcluster/build.sh x86_64-w64-mingw32"
     package: "taskcluster/package.sh"
   metadata:

taskcluster/tc-all-utils.sh

@@ -99,7 +99,7 @@ verify_bazel_rebuild()
 
   mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
 
-  cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
+  cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
 
   spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libdeepspeech.so' | wc -l)
   if [ "${spurious_rebuilds}" -ne 0 ]; then
@@ -108,13 +108,13 @@ verify_bazel_rebuild()
     if is_patched_bazel; then
       mkdir -p ${DS_ROOT_TASK}/DeepSpeech/ckd/ds ${DS_ROOT_TASK}/DeepSpeech/ckd/tf
       tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
-      tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/
+      tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/
 
       echo "Making a diff between CKD files"
       mkdir -p ${TASKCLUSTER_ARTIFACTS}
-      diff -urNw ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff
+      diff -urNw ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff
 
-      rm -fr ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
+      rm -fr ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
     else
       echo "Cannot get CKD information from release, please use patched Bazel"
     fi;

taskcluster/tc-all-vars.sh

@@ -49,7 +49,7 @@ export ANDROID_TMP_DIR=/data/local/tmp
 
 mkdir -p ${TASKCLUSTER_TMP_DIR} || true
 
-export DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/tf
+export DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow
 export DS_DSDIR=${DS_ROOT_TASK}/DeepSpeech/ds
 export DS_EXAMPLEDIR=${DS_ROOT_TASK}/DeepSpeech/examples
 

taskcluster/tc-build-utils.sh

@@ -173,26 +173,26 @@ do_deepspeech_npm_package()
 
 do_bazel_build()
 {
-  cd ${DS_ROOT_TASK}/DeepSpeech/tf
+  cd ${DS_TFDIR}
   eval "export ${BAZEL_ENV_FLAGS}"
 
   if is_patched_bazel; then
-    find ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T -
+    find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T -
   fi;
 
   bazel ${BAZEL_OUTPUT_USER_ROOT} build \
     -s --explain bazel_monolithic.log --verbose_explanations --experimental_strict_action_env --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
 
   if is_patched_bazel; then
-    find ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T -
+    find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T -
   fi;
 
-  verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/tf/bazel_monolithic.log"
+  verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_monolithic.log"
 }
 
 shutdown_bazel()
 {
-  cd ${DS_ROOT_TASK}/DeepSpeech/tf
+  cd ${DS_TFDIR}
   bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown
 }
 

taskcluster/tc-decision.py

@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from glob import glob
+from functools import reduce
+
+import json
+import jsone
+import os
+import sys
+import requests
+import slugid
+import yaml
+import subprocess
+
+import networkx as nx
+
+TASKS_ROOT = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])))
+TASKCLUSTER_API_BASEURL = 'http://taskcluster/queue/v1/task/%(task_id)s'
+
+def string_to_dict(sid, value):
+    parts = sid.split('.')
+
+    def pack(parts):
+        if len(parts) == 1:
+            return {parts[0]: value}
+        elif len(parts):
+            return {parts[0]: pack(parts[1:])}
+        return parts
+
+    return pack(parts)
+
+def merge_dicts(*dicts):
+    if not reduce(lambda x, y: isinstance(y, dict) and x, dicts, True):
+        raise TypeError("Object in *dicts not of type dict")
+    if len(dicts) < 2:
+        raise ValueError("Requires 2 or more dict objects")
+
+    def merge(a, b):
+        for d in set(a.keys()).union(b.keys()):
+            if d in a and d in b:
+                if type(a[d]) == type(b[d]):
+                    if not isinstance(a[d], dict):
+                        ret = list({a[d], b[d]})
+                        if len(ret) == 1: ret = ret[0]
+                        yield (d, sorted(ret))
+                    else:
+                        yield (d, dict(merge(a[d], b[d])))
+                else:
+                    raise TypeError("Conflicting key:value type assignment", type(a[d]), a[d], type(b[d]), b[d])
+            elif d in a:
+                yield (d, a[d])
+            elif d in b:
+                yield (d, b[d])
+            else:
+                raise KeyError
+
+    return reduce(lambda x, y: dict(merge(x, y)), dicts[1:], dicts[0])
+
+def taskcluster_event_context():
+    das_context = {}
+
+    # Pre-filterting
+    for k in os.environ.keys():
+        if k == 'GITHUB_HEAD_USER':
+            os.environ['GITHUB_HEAD_USER_LOGIN'] = os.environ[k]
+            del os.environ['GITHUB_HEAD_USER']
+
+    for k in os.environ.keys():
+        if k == 'TASK_ID':
+            parts = string_to_dict('taskcluster.taskGroupId', os.environ[k])
+            das_context = merge_dicts(das_context, parts)
+
+        if k.startswith('GITHUB_'):
+            parts = string_to_dict(k.lower().replace('_', '.').replace('github', 'event'), os.environ[k])
+            das_context = merge_dicts(das_context, parts)
+
+    return das_context
+
+def load_specific_contextFile(file):
+    specific_context = {}
+
+    try:
+        with open(os.path.join(TASKS_ROOT, file)) as src:
+            specific_context = yaml.load(src)
+
+        if specific_context is None:
+            specific_context = {}
+    except FileNotFoundError:
+        specific_context = {}
+
+    return specific_context
+
+def defaultValues_build_context():
+    return load_specific_contextFile('.build.yml')
+
+def shared_context():
+    return load_specific_contextFile('.shared.yml')
+
+def create_task_payload(build, base_context):
+    print('build', build)
+    build_type = os.path.splitext(os.path.basename(build))[0]
+
+    build_context = defaultValues_build_context()
+    with open(build) as src:
+        build_context['build'].update(yaml.load(src)['build'])
+
+    # Be able to use what has been defined in base_context
+    # e.g., the {${event.head.branch}}
+    build_context = jsone.render(build_context, base_context)
+    template_context = {
+        'taskcluster': {
+            'taskId': as_slugid(build_type)
+        },
+        'build_type': build_type
+    }
+
+    with open(os.path.join(TASKS_ROOT, build_context['build']['template_file'])) as src:
+        template = yaml.load(src)
+
+    contextes = merge_dicts({}, base_context, template_context, build_context)
+    for one_context in glob(os.path.join(TASKS_ROOT, '*.cyml')):
+        with open(one_context) as src:
+            contextes = merge_dicts(contextes, yaml.load(src))
+
+    return jsone.render(template, contextes)
+
+def send_task(t):
+    url = TASKCLUSTER_API_BASEURL % {'task_id': t['taskId']}
+    del t['taskId']
+
+    r = requests.put(url, json=t)
+
+    print(url, r.status_code)
+    if r.status_code != requests.codes.ok:
+        print(json.dumps(t, indent=2))
+        print(r.content)
+        print(json.loads(r.content.decode())['message'])
+
+    return r.status_code == requests.codes.ok
+
+slugids = {}
+def as_slugid(name):
+    if name not in slugids:
+        slugids[name] = slugid.nice().decode()
+        print('cache miss', name, slugids[name])
+    else:
+        print('cache hit', name, slugids[name])
+    return slugids[name]
+
+def to_int(x):
+    return int(x)
+
+def functions_context():
+    return {
+        'as_slugid': as_slugid,
+        'to_int': to_int
+    }
+
+def is_dry_run():
+    return (len(sys.argv) > 1) and (sys.argv[1] == '--dry')
+
+def should_run():
+    # Make a quick clone to fetch the last commit
+    try:
+        subprocess.check_call([
+            'git', 'clone', '--quiet', '-b', os.environ.get('GITHUB_HEAD_BRANCH'),
+            '--single-branch', os.environ.get('GITHUB_HEAD_REPO_URL'),
+            '--depth=1', '/tmp/ds-clone/'
+        ], env={'GIT_LFS_SKIP_SMUDGE': '1'})
+    except subprocess.CalledProcessError as e:
+        print("Error while git cloning:", e, file=sys.stderr)
+        return False
+
+    try:
+        git_msg = subprocess.check_output([
+            'git', '--git-dir=/tmp/ds-clone/.git/',
+            'log', '--format=%b', '-n', '1',
+            os.environ.get('GITHUB_HEAD_SHA')
+        ]).decode('utf-8').strip().upper()
+    except subprocess.CalledProcessError as e:
+        print("Error while git show:", e, file=sys.stderr)
+        return False
+
+    print('Commit message:', git_msg)
+
+    x_deepspeech = filter(lambda x: 'X-DEEPSPEECH:' in x, git_msg.split('\n'))
+    if len(list(filter(lambda x: 'NOBUILD' in x, x_deepspeech))) == 1:
+        print('Not running anything according to commit message')
+        return False
+
+    return True
+
+if __name__ == '__main__':
+    if not is_dry_run():
+        # We might want to NOT run in some cases
+        if not should_run():
+            sys.exit(0)
+
+    base_context = taskcluster_event_context()
+    base_context = merge_dicts(base_context, functions_context())
+    base_context = merge_dicts(base_context, shared_context())
+
+    root_task = base_context['taskcluster']['taskGroupId']
+
+    tasks_graph = nx.DiGraph()
+    tasks = {}
+
+    for build in glob(os.path.join(TASKS_ROOT, '*.yml')):
+        t = create_task_payload(build, base_context)
+
+        # We allow template to produce completely empty output
+        if not t:
+            continue
+
+        if 'dependencies' in t and len(t['dependencies']) > 0:
+            for dep in t['dependencies']:
+                tasks_graph.add_edge(t['taskId'], dep)
+        else:
+            tasks_graph.add_edge(t['taskId'], root_task)
+
+        tasks[t['taskId']] = t
+
+    for task in nx.dfs_postorder_nodes(tasks_graph):
+        # root_task is the task group and also the task id that is already
+        # running, so we don't have to schedule that
+        if task == root_task:
+            continue
+
+        t = tasks[task]
+        if is_dry_run():
+            print(json.dumps(t, indent=2))
+            continue
+
+        p = send_task(t)
+        if not p:
+            sys.exit(1)

taskcluster/tc-decision_reqs.txt

@@ -0,0 +1,5 @@
+json-e == 2.3.1
+networkx 
+pyaml
+requests
+slugid == 1.0.7

taskcluster/tc-schedule.sh

@@ -2,15 +2,13 @@
 
 set -ex
 
-# tc-decision.py assumes being at the root folder
+curdir=$(dirname "$0")/
-curdir=$(dirname "$0")/..
 
 pip3 install --quiet --user --upgrade pip
 
 export PATH=$HOME/.local/bin/:$PATH
 
-curl -L --silent https://raw.githubusercontent.com/lissyx/taskcluster-github-decision/${TC_DECISION_SHA}/requirements.txt | pip3 install --quiet --user --upgrade -r /dev/stdin
+pip3 install --quiet --user --upgrade -r ${curdir}/tc-decision_reqs.txt
-curl -L --silent https://raw.githubusercontent.com/lissyx/taskcluster-github-decision/${TC_DECISION_SHA}/tc-decision.py > ${curdir}/tc-decision.py
 
 # First, perform dry run for push and pull request
 # This should help us track merge failures in advance

taskcluster/test-win-cuda-opt-base.tyml

@@ -38,8 +38,6 @@ then:
         "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
         "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
       - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
-      - .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}"
-      - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
       - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
       - $let:
           extraSystemSetup: { $eval: strip(str(build.system_setup)) }
@@ -69,10 +67,6 @@ then:
         content:
           sha256: ${system.msys2.sha}
           url: ${system.msys2.url}
-      - file: filesystem-2020.02-3-x86_64.pkg.tar.xz
-        content:
-          sha256: ${system.msys2_filesystem_pkg.sha}
-          url: ${system.msys2_filesystem_pkg.url}
       - file: pyenv.tar.gz
         content:
           url: ${system.pyenv.win.url}

taskcluster/test-win-opt-base.tyml

@@ -40,8 +40,6 @@ then:
         "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
         "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
       - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
-      - .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}"
-      - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
       - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
       - $let:
           extraSystemSetup: { $eval: strip(str(build.system_setup)) }
@@ -71,10 +69,6 @@ then:
         content:
           sha256: ${system.msys2.sha}
           url: ${system.msys2.url}
-      - file: filesystem-2020.02-3-x86_64.pkg.tar.xz
-        content:
-          sha256: ${system.msys2_filesystem_pkg.sha}
-          url: ${system.msys2_filesystem_pkg.url}
       - file: pyenv.tar.gz
         content:
           url: ${system.pyenv.win.url}

taskcluster/tf_android-arm64-opt.yml

@@ -0,0 +1,16 @@
+build:
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.android_arm64.url}
+    artifact_namespace: ${system.tensorflow.android_arm64.namespace}
+  system_config:
+    >
+      ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh --android"
+    build: "taskcluster/tf_tc-build.sh --android-arm64"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 14400
+  metadata:
+    name: "TensorFlow Android ARM64"
+    description: "Building TensorFlow for Android ARM64, optimized version"

taskcluster/tf_android-armv7-opt.yml

@@ -0,0 +1,16 @@
+build:
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.android_armv7.url}
+    artifact_namespace: ${system.tensorflow.android_armv7.namespace}
+  system_config:
+    >
+      ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh --android"
+    build: "taskcluster/tf_tc-build.sh --android-armv7"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 14400
+  metadata:
+    name: "TensorFlow Android ARMv7"
+    description: "Building TensorFlow for Android ARMv7, optimized version"

taskcluster/tf_darwin-amd64-opt.yml

@@ -0,0 +1,18 @@
+build:
+  template_file: generic_tc_caching-darwin-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.darwin_amd64.url}
+    artifact_namespace: ${system.tensorflow.darwin_amd64.namespace}
+  generic:
+    workerType: "ds-macos-heavy"
+  system_config:
+    >
+      ${tensorflow.packages_macos.brew}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh"
+    build: "taskcluster/tf_tc-build.sh --osx"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 28800
+  metadata:
+    name: "TensorFlow OSX AMD64 CPU"
+    description: "Building TensorFlow for OSX AMD64, CPU only, optimized version"

taskcluster/tf_linux-amd64-cpu-opt.yml

@@ -0,0 +1,16 @@
+build:
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.linux_amd64_cpu.url}
+    artifact_namespace: ${system.tensorflow.linux_amd64_cpu.namespace}
+  system_config:
+    >
+      ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh"
+    build: "taskcluster/tf_tc-build.sh"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 14400
+  metadata:
+    name: "TensorFlow Linux AMD64 CPU"
+    description: "Building TensorFlow for Linux/AMD64, CPU only, optimized version"

taskcluster/tf_linux-amd64-gpu-opt.yml

@@ -0,0 +1,16 @@
+build:
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.linux_amd64_cuda.url}
+    artifact_namespace: ${system.tensorflow.linux_amd64_cuda.namespace}
+  system_config:
+    >
+      ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh --cuda"
+    build: "taskcluster/tf_tc-build.sh --gpu"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 14400
+  metadata:
+    name: "TensorFlow Linux AMD64 CUDA"
+    description: "Building TensorFlow for Linux/AMD64, CUDA-enabled, optimized version"

taskcluster/tf_linux-arm64-cpu-opt.yml

@@ -0,0 +1,16 @@
+build:
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.linux_arm64.url}
+    artifact_namespace: ${system.tensorflow.linux_arm64.namespace}
+  system_config:
+    >
+      ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh"
+    build: "taskcluster/tf_tc-build.sh --arm64"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 14400
+  metadata:
+    name: "TensorFlow Linux ARM64 Cortex-A53 CPU"
+    description: "Building TensorFlow for Linux ARM64 Cortex-A53, CPU only, optimized version"

taskcluster/tf_linux-rpi3-cpu-opt.yml

@@ -0,0 +1,16 @@
+build:
+  template_file: generic_tc_caching-linux-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.linux_armv7.url}
+    artifact_namespace: ${system.tensorflow.linux_armv7.namespace}
+  system_config:
+    >
+      ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh"
+    build: "taskcluster/tf_tc-build.sh --arm"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 14400
+  metadata:
+    name: "TensorFlow Linux RPi3/ARMv7 CPU"
+    description: "Building TensorFlow for Linux RPi3 ARMv7, CPU only, optimized version"

taskcluster/tf_tc-brew.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -ex
+
+if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then
+    echo "No TASKCLUSTER_TASK_DIR, aborting."
+    exit 1
+fi
+
+LOCAL_BREW="${TASKCLUSTER_TASK_DIR}/homebrew"
+export PATH=${LOCAL_BREW}/bin:$PATH
+export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/homebrew.logs/"
+export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/homebrew.cache/"
+
+# Never fail on pre-existing homebrew/ directory
+mkdir -p "${LOCAL_BREW}" || true
+mkdir -p "${HOMEBREW_CACHE}" || true
+
+# Make sure to verify there is a 'brew' binary there, otherwise install things.
+if [ ! -x "${LOCAL_BREW}/bin/brew" ]; then
+    curl -L https://github.com/Homebrew/brew/tarball/2.2.17 | tar xz --strip 1 -C "${LOCAL_BREW}"
+fi;
+
+echo "local brew list (should be empty) ..."
+brew list
+
+echo "local brew prefix ..."
+local_prefix=$(brew --prefix)
+echo "${local_prefix}"
+
+if [ "${LOCAL_BREW}" != "${local_prefix}" ]; then
+    echo "Weird state:"
+    echo "LOCAL_BREW=${LOCAL_BREW}"
+    echo "local_prefix=${local_prefix}"
+    exit 1
+fi;
+
+# coreutils, pyenv-virtualenv required for build of tensorflow
+all_pkgs="coreutils pyenv-virtualenv"
+
+for pkg in ${all_pkgs};
+do
+   (brew list --versions ${pkg} && brew upgrade ${pkg}) || brew install ${pkg}
+done;

taskcluster/tf_tc-build.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+set -ex
+
+source $(dirname $0)/tf_tc-vars.sh
+
+build_amd64=yes
+build_gpu=no
+build_android_arm=no
+build_android_arm64=no
+build_linux_arm=no
+build_linux_arm64=no
+
+if [ "$1" = "--gpu" ]; then
+    build_amd64=yes
+    build_gpu=yes
+    build_android_arm=no
+    build_android_arm64=no
+    build_linux_arm=no
+    build_linux_arm64=no
+fi
+
+if [ "$1" = "--arm" ]; then
+    build_amd64=yes
+    build_gpu=no
+    build_android_arm=no
+    build_android_arm64=no
+    build_linux_arm=yes
+    build_linux_arm64=no
+fi
+
+if [ "$1" = "--arm64" ]; then
+    build_amd64=yes
+    build_gpu=no
+    build_android_arm=no
+    build_android_arm64=no
+    build_linux_arm=no
+    build_linux_arm64=yes
+fi
+
+if [ "$1" = "--android-armv7" ]; then
+    build_amd64=no
+    build_gpu=no
+    build_android_arm=yes
+    build_android_arm64=no
+    build_linux_arm=no
+    build_linux_arm64=no
+fi
+
+if [ "$1" = "--android-arm64" ]; then
+    build_amd64=no
+    build_gpu=no
+    build_android_arm=no
+    build_android_arm64=yes
+    build_linux_arm=no
+    build_linux_arm64=no
+fi
+
+pushd ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/
+    BAZEL_BUILD="bazel ${BAZEL_OUTPUT_USER_ROOT} build -s --explain bazel_monolithic_tf.log --verbose_explanations --experimental_strict_action_env --config=monolithic"
+
+    # Start a bazel process to ensure reliability on Windows and avoid:
+    # FATAL: corrupt installation: file 'c:\builds\tc-workdir\.bazel_cache/install/6b1660721930e9d5f231f7d2a626209b/_embedded_binaries/build-runfiles.exe' missing.
+    bazel ${BAZEL_OUTPUT_USER_ROOT} info
+
+    # Force toolchain sync (useful on macOS ?)
+    bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure
+
+    if [ "${build_amd64}" = "yes" ]; then
+        # Pure amd64 CPU-only build
+        if [ "${OS}" = "${TC_MSYS_VERSION}" -a  "${build_gpu}" = "no" ]; then
+            echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
+        elif [ "${build_gpu}" = "no" -a "${build_linux_arm}" = "no" -a "${build_linux_arm64}" = "no" ]; then
+            echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
+        fi
+
+        # Cross RPi3 CPU-only build
+        if [ "${build_linux_arm}" = "yes" ]; then
+            echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
+        fi
+
+        # Cross ARM64 Cortex-A53 build
+        if [ "${build_linux_arm64}" = "yes" ]; then
+            echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
+        fi
+
+        # Pure amd64 GPU-enabled build
+        if [ "${build_gpu}" = "yes" ]; then
+            eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} -c opt ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API}
+        fi
+    fi
+
+    if [ "${build_android_arm}" = "yes" ]; then
+        echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
+    fi;
+
+    if [ "${build_android_arm64}" = "yes" ]; then
+        echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
+    fi;
+
+    if [ $? -ne 0 ]; then
+        # There was a failure, just account for it.
+        echo "Build failure, please check the output above. Exit code was: $?"
+        return 1
+    fi
+
+    bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown
+popd

taskcluster/tf_tc-package.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -xe
+
+source $(dirname $0)/tf_tc-vars.sh
+
+mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
+
+cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_*.log ${TASKCLUSTER_ARTIFACTS}
+
+OUTPUT_ROOT="${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin"
+
+for output_bin in                                                            \
+    tensorflow/lite/experimental/c/libtensorflowlite_c.so                    \
+    tensorflow/tools/graph_transforms/transform_graph                        \
+    tensorflow/tools/graph_transforms/summarize_graph                        \
+    tensorflow/tools/benchmark/benchmark_model                               \
+    tensorflow/contrib/util/convert_graphdef_memmapped_format                \
+    tensorflow/lite/toco/toco;
+do
+    if [ -f "${OUTPUT_ROOT}/${output_bin}" ]; then
+        cp ${OUTPUT_ROOT}/${output_bin} ${TASKCLUSTER_ARTIFACTS}/
+    fi;
+done;
+
+if [ -f "${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model" ]; then
+    cp ${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model ${TASKCLUSTER_ARTIFACTS}/lite_benchmark_model
+fi
+
+# It seems that bsdtar and gnutar are behaving a bit differently on the way
+# they deal with --exclude="./public/*" ; this caused ./DeepSpeech/tensorflow/core/public/
+# to be ditched when we just wanted to get rid of ./public/ on OSX.
+# Switching to gnutar (already needed for the --transform on DeepSpeech tasks)
+# does the trick.
+TAR_EXCLUDE="--exclude=./dls/*"
+if [ "${OS}" = "Darwin" ]; then
+    TAR_EXCLUDE="--exclude=./dls/* --exclude=./public/* --exclude=./generic-worker/* --exclude=./homebrew/* --exclude=./homebrew.cache/* --exclude=./homebrew.logs/*"
+fi;
+
+# Make a tar of
+#  - /home/build-user/ (linux
+#  - /Users/build-user/TaskCluster/HeavyTasks/X/ (OSX)
+#  - C:\builds\tc-workdir\ (windows)
+
+if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    export PATH=$PATH:'/c/Program Files/7-Zip/'
+    pushd ${DS_ROOT_TASK}
+        7z a '-xr!.\dls\' '-xr!.\tmp\' '-xr!.\msys64\' -snl -snh -so home.tar . | 7z a -si ${TASKCLUSTER_ARTIFACTS}/home.tar.xz
+    popd
+else
+    ${TAR} -C ${DS_ROOT_TASK} ${TAR_EXCLUDE} -cf - . | ${XZ} > ${TASKCLUSTER_ARTIFACTS}/home.tar.xz
+fi
+
+if [ "${OS}" = "Linux" ]; then
+    SHA_SUM_GEN="sha256sum"
+elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    SHA_SUM_GEN="sha256sum"
+elif [ "${OS}" = "Darwin" ]; then
+    SHA_SUM_GEN="shasum -a 256"
+fi;
+
+${SHA_SUM_GEN} ${TASKCLUSTER_ARTIFACTS}/* > ${TASKCLUSTER_ARTIFACTS}/checksums.txt

taskcluster/tf_tc-pip.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+# Taken from https://www.tensorflow.org/install/source
+# Only future is needed for our builds, as we don't build the Python package
+pip install -U --user future==0.17.1

taskcluster/tf_tc-setup.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+
+set -ex
+
+source $(dirname $0)/tf_tc-vars.sh
+
+install_cuda=
+if [ "$1" = "--cuda" ]; then
+    install_cuda=yes
+fi
+
+install_android=
+if [ "$1" = "--android" ]; then
+    install_android=yes
+fi
+
+# $1 url
+# $2 sha256
+download()
+{
+    fname=`basename $1`
+
+    ${WGET} $1 -O ${DS_ROOT_TASK}/dls/$fname && echo "$2  ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} -
+}
+
+# Download stuff
+mkdir -p ${DS_ROOT_TASK}/dls || true
+download $BAZEL_URL $BAZEL_SHA256
+
+if [ ! -z "${install_cuda}" ]; then
+    download $CUDA_URL $CUDA_SHA256
+    download $CUDNN_URL $CUDNN_SHA256
+fi;
+
+if [ ! -z "${install_android}" ]; then
+    download $ANDROID_NDK_URL $ANDROID_NDK_SHA256
+    download $ANDROID_SDK_URL $ANDROID_SDK_SHA256
+fi;
+
+# For debug
+ls -hal ${DS_ROOT_TASK}/dls/
+
+# Install Bazel in ${DS_ROOT_TASK}/bin
+BAZEL_INSTALL_FILENAME=$(basename "${BAZEL_URL}")
+if [ "${OS}" = "Linux" ]; then
+    BAZEL_INSTALL_FLAGS="--user"
+elif [ "${OS}" = "Darwin" ]; then
+    BAZEL_INSTALL_FLAGS="--bin=${DS_ROOT_TASK}/bin --base=${DS_ROOT_TASK}/.bazel"
+fi;
+mkdir -p ${DS_ROOT_TASK}/bin || true
+pushd ${DS_ROOT_TASK}/bin
+    if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+        cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel.exe
+    else
+        /bin/bash ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${BAZEL_INSTALL_FLAGS}
+    fi
+popd
+
+# For debug
+bazel version
+
+bazel shutdown
+
+if [ ! -z "${install_cuda}" ]; then
+    # Install CUDA and CuDNN
+    mkdir -p ${DS_ROOT_TASK}/DeepSpeech/CUDA/ || true
+    pushd ${DS_ROOT_TASK}
+        CUDA_FILE=`basename ${CUDA_URL}`
+        PERL5LIB=. sh ${DS_ROOT_TASK}/dls/${CUDA_FILE} --silent --override --toolkit --toolkitpath=${DS_ROOT_TASK}/DeepSpeech/CUDA/ --defaultroot=${DS_ROOT_TASK}/DeepSpeech/CUDA/
+
+        CUDNN_FILE=`basename ${CUDNN_URL}`
+        tar xvf ${DS_ROOT_TASK}/dls/${CUDNN_FILE} --strip-components=1 -C ${DS_ROOT_TASK}/DeepSpeech/CUDA/
+    popd
+
+    LD_LIBRARY_PATH=${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/:${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/:$LD_LIBRARY_PATH
+    export LD_LIBRARY_PATH
+
+    # We might lack libcuda.so.1 symlink, let's fix as upstream does:
+    # https://github.com/tensorflow/tensorflow/pull/13811/files?diff=split#diff-2352449eb75e66016e97a591d3f0f43dR96
+    if [ ! -h "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so.1" ]; then
+        ln -s "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so" "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so.1"
+    fi;
+
+else
+    echo "No CUDA/CuDNN to install"
+fi
+
+if [ ! -z "${install_android}" ]; then
+    mkdir -p ${DS_ROOT_TASK}/DeepSpeech/Android/SDK || true
+    ANDROID_NDK_FILE=`basename ${ANDROID_NDK_URL}`
+    ANDROID_SDK_FILE=`basename ${ANDROID_SDK_URL}`
+
+    pushd ${DS_ROOT_TASK}/DeepSpeech/Android
+        unzip ${DS_ROOT_TASK}/dls/${ANDROID_NDK_FILE}
+    popd
+
+    pushd ${DS_ROOT_TASK}/DeepSpeech/Android/SDK
+        unzip ${DS_ROOT_TASK}/dls/${ANDROID_SDK_FILE}
+        yes | ./tools/bin/sdkmanager --licenses
+        ./tools/bin/sdkmanager --update
+        ./tools/bin/sdkmanager --install "platforms;android-16" "build-tools;28.0.3"
+    popd
+fi
+
+mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
+
+pushd ${DS_ROOT_TASK}/DeepSpeech/ds/
+    git submodule --quiet sync tensorflow/ && git submodule --quiet update --init tensorflow/
+popd
+
+
+# Taken from https://www.tensorflow.org/install/source
+# Only future is needed for our builds, as we don't build the Python package
+pip install -U --user future==0.17.1 || true

taskcluster/tf_tc-vars.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+set -ex
+
+export OS=$(uname)
+if [ "${OS}" = "Linux" ]; then
+    export DS_ROOT_TASK=$(/usr/bin/realpath "${HOME}")
+
+    BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-linux-x86_64.sh
+    BAZEL_SHA256=2fbdc9c0e3d376697caf0ee3673b7c9475214068c55a01b9744891e131f90b87
+
+    CUDA_URL=http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run
+    CUDA_SHA256=e7c22dc21278eb1b82f34a60ad7640b41ad3943d929bebda3008b72536855d31
+
+    # From https://gitlab.com/nvidia/cuda/blob/centos7/10.1/devel/cudnn7/Dockerfile
+    CUDNN_URL=http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.0/cudnn-10.1-linux-x64-v7.6.0.64.tgz
+    CUDNN_SHA256=e956c6f9222fcb867a10449cfc76dee5cfd7c7531021d95fe9586d7e043b57d7
+
+    ANDROID_NDK_URL=https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip
+    ANDROID_NDK_SHA256=4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd
+
+    ANDROID_SDK_URL=https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip
+    ANDROID_SDK_SHA256=92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9
+
+    SHA_SUM="sha256sum -c --strict"
+    WGET=/usr/bin/wget
+    TAR=tar
+    XZ="pixz -9"
+elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    if [ -z "${TASKCLUSTER_TASK_DIR}" -o -z "${TASKCLUSTER_ARTIFACTS}" ]; then
+        echo "Inconsistent Windows setup: missing some vars."
+        echo "TASKCLUSTER_TASK_DIR=${TASKCLUSTER_TASK_DIR}"
+        echo "TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS}"
+        exit 1
+    fi;
+
+    # Re-export with cygpath to make sure it is sane, otherwise it might trigger
+    # unobvious failures with cp etc.
+    export TASKCLUSTER_TASK_DIR="$(cygpath ${TASKCLUSTER_TASK_DIR})"
+    export TASKCLUSTER_ARTIFACTS="$(cygpath ${TASKCLUSTER_ARTIFACTS})"
+
+    export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
+    export BAZEL_VC='C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC'
+    export BAZEL_SH='C:\builds\tc-workdir\msys64\usr\bin\bash'
+    export TC_WIN_BUILD_PATH='C:\builds\tc-workdir\msys64\usr\bin;C:\Python36'
+    export MSYS2_ARG_CONV_EXCL='//'
+
+    mkdir -p ${TASKCLUSTER_TASK_DIR}/tmp/
+    export TEMP=${TASKCLUSTER_TASK_DIR}/tmp/
+    export TMP=${TASKCLUSTER_TASK_DIR}/tmp/
+
+    BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-windows-x86_64.exe
+    BAZEL_SHA256=cc7b3ff6f4bfd6bc2121a80656afec66ee57713e8b88e9d2fb58b4eddf271268
+
+    CUDA_INSTALL_DIRECTORY=$(cygpath 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1')
+
+    SHA_SUM="sha256sum -c --strict"
+    WGET=wget
+    TAR=/usr/bin/tar.exe
+    XZ="xz -9 -T0"
+elif [ "${OS}" = "Darwin" ]; then
+    if [ -z "${TASKCLUSTER_TASK_DIR}" -o -z "${TASKCLUSTER_ARTIFACTS}" ]; then
+        echo "Inconsistent OSX setup: missing some vars."
+        echo "TASKCLUSTER_TASK_DIR=${TASKCLUSTER_TASK_DIR}"
+        echo "TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS}"
+        exit 1
+    fi;
+
+    export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
+
+    BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-darwin-x86_64.sh
+    BAZEL_SHA256=c675fa27d99a3114d681db10eb03ded547c40f702b2048c99b8f4ea8e89b9356
+
+    SHA_SUM="shasum -a 256 -c"
+    WGET=wget
+    TAR=gtar
+    XZ="pixz -9"
+fi;
+
+# /tmp/artifacts for docker-worker on linux,
+# and task subdir for generic-worker on osx
+export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts}
+
+### Define variables that needs to be exported to other processes
+
+PATH=${DS_ROOT_TASK}/bin:$PATH
+if [ "${OS}" = "Darwin" ]; then
+    PATH=${DS_ROOT_TASK}/homebrew/bin/:${DS_ROOT_TASK}/homebrew/opt/node@10/bin:$PATH
+fi;
+export PATH
+
+if [ "${OS}" = "Linux" ]; then
+    export LD_LIBRARY_PATH=${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/:${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/:$LD_LIBRARY_PATH
+    export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/
+    export ANDROID_NDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/android-ndk-r18b/
+fi;
+
+export TF_ENABLE_XLA=0
+if [ "${OS}" = "Linux" ]; then
+    TF_NEED_JEMALLOC=1
+elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    TF_NEED_JEMALLOC=0
+elif [ "${OS}" = "Darwin" ]; then
+    TF_NEED_JEMALLOC=0
+fi;
+export TF_NEED_JEMALLOC
+export TF_NEED_OPENCL_SYCL=0
+export TF_NEED_MKL=0
+export TF_NEED_VERBS=0
+export TF_NEED_MPI=0
+export TF_NEED_IGNITE=0
+export TF_NEED_GDR=0
+export TF_NEED_NGRAPH=0
+export TF_DOWNLOAD_CLANG=0
+export TF_SET_ANDROID_WORKSPACE=0
+export TF_NEED_TENSORRT=0
+export TF_NEED_ROCM=0
+
+# This should be gcc-5, hopefully. CUDA and TensorFlow might not be happy, otherwise.
+export GCC_HOST_COMPILER_PATH=/usr/bin/gcc
+
+if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    export PYTHON_BIN_PATH=C:/Python36/python.exe
+else
+    export PYTHON_BIN_PATH=/usr/bin/python2.7
+fi
+
+## Below, define or export some build variables
+
+# Enable some SIMD support. Limit ourselves to what Tensorflow needs.
+# Also ensure to not require too recent CPU: AVX2/FMA introduced by:
+#  - Intel with Haswell (2013)
+#  - AMD with Excavator (2015)
+# For better compatibility, AVX ony might be better.
+#
+# Build for generic amd64 platforms, no device-specific optimization
+# See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for targetting specific CPUs
+
+if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    CC_OPT_FLAGS="/arch:AVX"
+else
+    CC_OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx"
+fi
+BAZEL_OPT_FLAGS=""
+for flag in ${CC_OPT_FLAGS};
+do
+    BAZEL_OPT_FLAGS="${BAZEL_OPT_FLAGS} --copt=${flag}"
+done;
+
+export CC_OPT_FLAGS
+
+BAZEL_OUTPUT_CACHE_DIR="${DS_ROOT_TASK}/.bazel_cache/"
+BAZEL_OUTPUT_CACHE_INSTANCE="${BAZEL_OUTPUT_CACHE_DIR}/output/"
+mkdir -p ${BAZEL_OUTPUT_CACHE_INSTANCE} || true
+
+# We need both to ensure stable path ; default value for output_base is some
+# MD5 value.
+BAZEL_OUTPUT_USER_ROOT="--output_user_root ${BAZEL_OUTPUT_CACHE_DIR} --output_base ${BAZEL_OUTPUT_CACHE_INSTANCE}"
+export BAZEL_OUTPUT_USER_ROOT
+
+NVCC_COMPUTE="3.5"
+
+### Define build parameters/env variables that we will re-ues in sourcing scripts.
+if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_PATHS=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\""
+else
+    TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${DS_ROOT_TASK}/DeepSpeech/CUDA\" TF_CUDA_PATHS=\"${DS_ROOT_TASK}/DeepSpeech/CUDA\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\""
+fi
+BAZEL_ARM_FLAGS="--config=rpi3 --config=rpi3_opt --copt=-DTFLITE_WITH_RUY_GEMV"
+BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_WITH_RUY_GEMV"
+BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV"
+BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV"
+BAZEL_CUDA_FLAGS="--config=cuda"
+
+if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
+    # Somehow, even with Python being in the PATH, Bazel on windows struggles
+    # with '/usr/bin/env python' ...
+    #
+    # We also force TMP/TEMP otherwise Bazel will pick default Windows one
+    # under %USERPROFILE%\AppData\Local\Temp and with 8.3 file format convention
+    # it messes with cxx_builtin_include_directory
+    BAZEL_EXTRA_FLAGS="--action_env=PATH=${TC_WIN_BUILD_PATH} --action_env=TEMP=${TEMP} --action_env=TMP=${TMP}"
+else
+    BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl --copt=-fvisibility=hidden"
+fi
+
+### Define build targets that we will re-ues in sourcing scripts.
+BUILD_TARGET_LIB_CPP_API="//tensorflow:tensorflow_cc"
+BUILD_TARGET_GRAPH_TRANSFORMS="//tensorflow/tools/graph_transforms:transform_graph"
+BUILD_TARGET_GRAPH_SUMMARIZE="//tensorflow/tools/graph_transforms:summarize_graph"
+BUILD_TARGET_GRAPH_BENCHMARK="//tensorflow/tools/benchmark:benchmark_model"
+#BUILD_TARGET_CONVERT_MMAP="//tensorflow/contrib/util:convert_graphdef_memmapped_format"
+BUILD_TARGET_TOCO="//tensorflow/lite/toco:toco"
+BUILD_TARGET_LITE_BENCHMARK="//tensorflow/lite/tools/benchmark:benchmark_model"
+BUILD_TARGET_LITE_LIB="//tensorflow/lite/c:libtensorflowlite_c.so"

taskcluster/tf_win-amd64-cpu-opt.yml

@@ -0,0 +1,16 @@
+build:
+  template_file: generic_tc_caching-win-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.win_amd64_cpu.url}
+    artifact_namespace: ${system.tensorflow.win_amd64_cpu.namespace}
+  system_config:
+    >
+      ${tensorflow.packages_win.pacman} && ${tensorflow.packages_win.msys64}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh"
+    build: "taskcluster/tf_tc-build.sh"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 14400
+  metadata:
+    name: "TensorFlow Windows AMD64 CPU"
+    description: "Building TensorFlow for Windows AMD64, CPU only, optimized version"

taskcluster/tf_win-amd64-gpu-opt.yml

@@ -0,0 +1,16 @@
+build:
+  template_file: generic_tc_caching-win-opt-base.tyml
+  cache:
+    artifact_url: ${system.tensorflow.win_amd64_cuda.url}
+    artifact_namespace: ${system.tensorflow.win_amd64_cuda.namespace}
+  system_config:
+    >
+      ${tensorflow.packages_win.pacman} && ${tensorflow.packages_win.msys64}
+  scripts:
+    setup: "taskcluster/tf_tc-setup.sh"
+    build: "taskcluster/tf_tc-build.sh --gpu"
+    package: "taskcluster/tf_tc-package.sh"
+  maxRunTime: 14400
+  metadata:
+    name: "TensorFlow Windows AMD64 CUDA"
+    description: "Building TensorFlow for Windows AMD64, CUDA, optimized version"

taskcluster/win-amd64-cpu-opt.yml

@@ -8,7 +8,8 @@ build:
     - "node-gyp-cache"
     - "swig-win-amd64"
     - "pyenv-win-amd64"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz"
+    - "tf_win-amd64-cpu-opt"
+  tensorflow: ${system.tensorflow.win_amd64_cpu.url}
   scripts:
     build: "taskcluster/win-build.sh"
     package: "taskcluster/win-package.sh"

taskcluster/win-amd64-ctc-opt.yml

@@ -4,11 +4,12 @@ build:
     - "swig-win-amd64"
     - "node-gyp-cache"
     - "pyenv-win-amd64"
+    - "tf_win-amd64-cpu-opt"
   routes:
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win-ctc"
     - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win-ctc"
     - "index.project.deepspeech.deepspeech.native_client.win-ctc.${event.head.sha}"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz"
+  tensorflow: ${system.tensorflow.win_amd64_cpu.url}
   scripts:
     build: 'taskcluster/decoder-build.sh'
     package: 'taskcluster/decoder-package.sh'

taskcluster/win-amd64-gpu-opt.yml

@@ -8,7 +8,8 @@ build:
     - "node-gyp-cache"
     - "swig-win-amd64"
     - "pyenv-win-amd64"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win-cuda/artifacts/public/home.tar.xz"
+    - "tf_win-amd64-gpu-opt"
+  tensorflow: ${system.tensorflow.win_amd64_cuda.url}
   scripts:
     build: "taskcluster/win-build.sh --cuda"
     package: "taskcluster/win-package.sh"

taskcluster/win-amd64-tflite-opt.yml

@@ -8,7 +8,8 @@ build:
     - "node-gyp-cache"
     - "swig-win-amd64"
     - "pyenv-win-amd64"
-  tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz"
+    - "tf_win-amd64-cpu-opt"
+  tensorflow: ${system.tensorflow.win_amd64_cpu.url}
   scripts:
     build: "taskcluster/win-build.sh --tflite"
     package: "taskcluster/win-package.sh"

taskcluster/win-build.sh

@@ -6,7 +6,7 @@ package_option=$1
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
+source $(dirname "$0")/tf_tc-vars.sh
 
 BAZEL_TARGETS="
 //native_client:libdeepspeech.so
@@ -31,7 +31,7 @@ SYSTEM_TARGET=host-win
 do_bazel_build
 
 if [ "${package_option}" = "--cuda" ]; then
-    cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/libdeepspeech.so.if.lib
+    cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin/native_client/libdeepspeech.so.if.lib
 fi
 
 export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin:'/c/Program Files/nodejs/'

taskcluster/win-opt-base.tyml

@@ -40,8 +40,6 @@ payload:
       "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
       "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
     - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
-    - .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}"
-    - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
     - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
     - echo .\msys64\usr\bin\bash.exe --login -cxe "
       export LC_ALL=C &&
@@ -54,9 +52,10 @@ payload:
       (7z x -txz -so $USERPROFILE/home.tar.xz | 7z x -aoa -ttar -si ) &&
       git clone --quiet $EXAMPLES_CLONE_URL $TASKCLUSTER_TASK_DIR/DeepSpeech/examples &&
       cd $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && git checkout --quiet $EXAMPLES_CHECKOUT_TARGET &&
-      git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
+      cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} &&
-      cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
+      git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ &&
-      ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
+      (rm $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tensorflow/native_client || true) &&
+      ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tensorflow/native_client &&
       cd $TASKCLUSTER_TASK_DIR &&
       (mkdir pyenv-root/ && 7z x -so $USERPROFILE/pyenv.tar.gz | 7z x -opyenv-root/ -aoa -ttar -si ) &&
       pacman --noconfirm -S tar make &&
@@ -75,10 +74,6 @@ payload:
       content:
         sha256: ${system.msys2.sha}
         url: ${system.msys2.url}
-    - file: filesystem-2020.02-3-x86_64.pkg.tar.xz
-      content:
-        sha256: ${system.msys2_filesystem_pkg.sha}
-        url: ${system.msys2_filesystem_pkg.url}
     - file: home.tar.xz
       content:
         url: ${build.tensorflow}

taskcluster/win-package.sh

@@ -6,7 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh
 
 mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
 
-cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
+cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
 
 package_native_client "native_client.tar.xz"
 

tensorflow

@@ -0,0 +1 @@
+Subproject commit 7ead55807a2ded84c107720ebca61e6285e2c239