Use TensorFlow as a submodule
This commit is contained in:
Alexandre Lissy
parent
be006da2d2
commit
80ee63fac6
|
|
@ -2,3 +2,6 @@
|
|||
path = doc/examples
|
||||
url = https://github.com/mozilla/DeepSpeech-examples.git
|
||||
branch = master
|
||||
[submodule "tensorflow"]
|
||||
path = tensorflow
|
||||
url = https://github.com/mozilla/tensorflow.git
|
||||
|
|
|
|||
|
|
@ -53,11 +53,6 @@ RUN dpkg -i bazel_*.deb
|
|||
|
||||
# >> START Configure Tensorflow Build
|
||||
|
||||
# Clone TensorFlow from Mozilla repo
|
||||
RUN git clone https://github.com/mozilla/tensorflow/
|
||||
WORKDIR /tensorflow
|
||||
RUN git checkout r2.2
|
||||
|
||||
# GPU Environment Setup
|
||||
ENV TF_NEED_ROCM 0
|
||||
ENV TF_NEED_OPENCL_SYCL 0
|
||||
|
|
@ -116,16 +111,15 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
|
|||
|
||||
WORKDIR /
|
||||
|
||||
RUN git clone $DEEPSPEECH_REPO
|
||||
RUN git clone --recursive $DEEPSPEECH_REPO
|
||||
WORKDIR /DeepSpeech
|
||||
RUN git checkout $DEEPSPEECH_SHA
|
||||
|
||||
# Link DeepSpeech native_client libs to tf folder
|
||||
RUN ln -s /DeepSpeech/native_client /tensorflow
|
||||
RUN git submodule sync tensorflow/
|
||||
RUN git submodule update --init tensorflow/
|
||||
|
||||
# >> START Build and bind
|
||||
|
||||
WORKDIR /tensorflow
|
||||
WORKDIR /DeepSpeech/tensorflow
|
||||
|
||||
# Fix for not found script https://github.com/tensorflow/tensorflow/issues/471
|
||||
RUN ./configure
|
||||
|
|
@ -158,10 +152,10 @@ RUN bazel build \
|
|||
--action_env=LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
|
||||
|
||||
# Copy built libs to /DeepSpeech/native_client
|
||||
RUN cp /tensorflow/bazel-bin/native_client/libdeepspeech.so /DeepSpeech/native_client/
|
||||
RUN cp bazel-bin/native_client/libdeepspeech.so /DeepSpeech/native_client/
|
||||
|
||||
# Build client.cc and install Python client and decoder bindings
|
||||
ENV TFDIR /tensorflow
|
||||
ENV TFDIR /DeepSpeech/tensorflow
|
||||
|
||||
RUN nproc
|
||||
|
||||
|
|
|
|||
|
|
@ -4,10 +4,8 @@ Building DeepSpeech Binaries
|
|||
|
||||
If you'd like to build the DeepSpeech binaries yourself, you'll need the following pre-requisites downloaded and installed:
|
||||
|
||||
|
||||
* `Mozilla's TensorFlow r2.2 branch <https://github.com/mozilla/tensorflow/tree/r2.2>`_
|
||||
* `Bazel 2.0.0 <https://github.com/bazelbuild/bazel/releases/tag/2.0.0>`_
|
||||
* `General TensorFlow requirements <https://www.tensorflow.org/install/install_sources>`_
|
||||
* `General TensorFlow r2.2 requirements <https://www.tensorflow.org/install/source#tested_build_configurations>`_
|
||||
* `libsox <https://sourceforge.net/projects/sox/>`_
|
||||
|
||||
It is required to use our fork of TensorFlow since it includes fixes for common problems encountered when building the native client files.
|
||||
|
|
@ -28,15 +26,16 @@ If you follow these instructions, you should compile your own binaries of DeepSp
|
|||
|
||||
For more information on configuring TensorFlow, read the docs up to the end of `"Configure the Build" <https://www.tensorflow.org/install/source#configure_the_build>`_.
|
||||
|
||||
TensorFlow: Clone & Checkout
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
Checkout source code
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Clone our fork of TensorFlow and checkout the correct version:
|
||||
Clone DeepSpeech source code (TensorFlow will come as a submdule):
|
||||
|
||||
.. code-block::
|
||||
|
||||
git clone https://github.com/mozilla/tensorflow.git
|
||||
git checkout origin/r2.2
|
||||
git clone https://github.com/mozilla/DeepSpeech.git
|
||||
git submodule sync tensorflow/
|
||||
git submodule update --init tensorflow/
|
||||
|
||||
Bazel: Download & Install
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
|
@ -57,16 +56,16 @@ Compile DeepSpeech
|
|||
------------------
|
||||
|
||||
Compile ``libdeepspeech.so``
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Within your TensorFlow checkout, create a symbolic link to the DeepSpeech ``native_client`` directory. Assuming DeepSpeech and TensorFlow checkouts are in the same directory, do:
|
||||
Within your TensorFlow directory, there should be a symbolic link to the DeepSpeech ``native_client`` directory. If it is not present, create it with the follow command:
|
||||
|
||||
.. code-block::
|
||||
|
||||
cd tensorflow
|
||||
ln -s ../DeepSpeech/native_client ./
|
||||
ln -s ../native_client
|
||||
|
||||
You can now use Bazel to build the main DeepSpeech library, ``libdeepspeech.so``\ . Add ``--config=cuda`` if you want a CUDA build.
|
||||
You can now use Bazel to build the main DeepSpeech library, ``libdeepspeech.so``. Add ``--config=cuda`` if you want a CUDA build.
|
||||
|
||||
.. code-block::
|
||||
|
||||
|
|
@ -77,11 +76,10 @@ The generated binaries will be saved to ``bazel-bin/native_client/``.
|
|||
Compile Language Bindings
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Now, ``cd`` into the ``DeepSpeech/native_client`` directory and use the ``Makefile`` to build all the language bindings (C++ client, Python package, Nodejs package, etc.). Set the environment variable ``TFDIR`` to point to your TensorFlow checkout.
|
||||
Now, ``cd`` into the ``DeepSpeech/native_client`` directory and use the ``Makefile`` to build all the language bindings (C++ client, Python package, Nodejs package, etc.).
|
||||
|
||||
.. code-block::
|
||||
|
||||
TFDIR=~/tensorflow
|
||||
cd ../DeepSpeech/native_client
|
||||
make deepspeech
|
||||
|
||||
|
|
@ -191,11 +189,11 @@ Building the ``deepspeech`` binary will happen through ``ndk-build`` (ARMv7):
|
|||
.. code-block::
|
||||
|
||||
cd ../DeepSpeech/native_client
|
||||
$ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../../tensorflow/ TARGET_ARCH_ABI=armeabi-v7a
|
||||
$ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../tensorflow/ TARGET_ARCH_ABI=armeabi-v7a
|
||||
|
||||
And (ARM64):
|
||||
|
||||
.. code-block::
|
||||
|
||||
cd ../DeepSpeech/native_client
|
||||
$ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../../tensorflowx/ TARGET_ARCH_ABI=arm64-v8a
|
||||
$ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../tensorflow/ TARGET_ARCH_ABI=arm64-v8a
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
NC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
|
||||
|
||||
TARGET ?= host
|
||||
TFDIR ?= $(abspath $(NC_DIR)/../../tensorflow)
|
||||
TFDIR ?= $(abspath $(NC_DIR)/../tensorflow)
|
||||
PREFIX ?= /usr/local
|
||||
SO_SEARCH ?= $(TFDIR)/bazel-bin/
|
||||
|
||||
|
|
|
|||
|
|
@ -43,36 +43,34 @@ We highly recommend sticking to the recommended versions of CUDA/cuDNN in order
|
|||
Getting the code
|
||||
----------------
|
||||
|
||||
We need to clone ``mozilla/DeepSpeech`` and ``mozilla/tensorflow``.
|
||||
We need to clone ``mozilla/DeepSpeech``.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
git clone https://github.com/mozilla/DeepSpeech
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
git clone --branch r2.2 https://github.com/mozilla/tensorflow
|
||||
git submodule sync tensorflow/
|
||||
git submodule update --init tensorflow/
|
||||
|
||||
Configuring the paths
|
||||
---------------------
|
||||
|
||||
We need to create a symbolic link, for this example let's suppose that we cloned into ``D:\cloned`` and now the structure looks like:
|
||||
There should already be a symbolic link, for this example let's suppose that we cloned into ``D:\cloned`` and now the structure looks like:
|
||||
|
||||
.. code-block::
|
||||
|
||||
.
|
||||
├── D:\
|
||||
│ ├── cloned # Contains DeepSpeech and tensorflow side by side
|
||||
│ │ ├── DeepSpeech # Root of the cloned DeepSpeech
|
||||
│ │ ├── tensorflow # Root of the cloned Mozilla's tensorflow
|
||||
│ │ └── DeepSpeech # Root of the cloned DeepSpeech
|
||||
│ │ ├── tensorflow # Root of the cloned Mozilla's tensorflow
|
||||
└── ...
|
||||
|
||||
|
||||
Change your path accordingly to your path structure, for the structure above we are going to use the following command:
|
||||
Change your path accordingly to your path structure, for the structure above we are going to use the following command if the symbolic link does not exists:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
mklink /d "D:\cloned\tensorflow\native_client" "D:\cloned\DeepSpeech\native_client"
|
||||
mklink /d "D:\cloned\DeepSpeech\tensorflow\native_client" "D:\cloned\DeepSpeech\native_client"
|
||||
|
||||
Adding environment variables
|
||||
----------------------------
|
||||
|
|
@ -82,7 +80,7 @@ After you have installed the requirements there are few environment variables th
|
|||
MSYS2 paths
|
||||
~~~~~~~~~~~
|
||||
|
||||
For MSYS2 we need to add ``bin`` directory, if you installed in the default route the path that we need to add should looks like ``C:\msys64\usr\bin``. Now we can run ``pacman``\ :
|
||||
For MSYS2 we need to add ``bin`` directory, if you installed in the default route the path that we need to add should looks like ``C:\msys64\usr\bin``. Now we can run ``pacman``:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
|
|
@ -120,7 +118,7 @@ Building the native_client
|
|||
|
||||
There's one last command to run before building, you need to run the `configure.py <https://github.com/mozilla/tensorflow/blob/master/configure.py>`_ inside ``tensorflow`` cloned directory.
|
||||
|
||||
At this point we are ready to start building the ``native_client``\ , go to ``tensorflow`` directory that you cloned, following our examples should be ``D:\cloned\tensorflow``.
|
||||
At this point we are ready to start building the ``native_client``, go to ``tensorflow`` sub-directory, following our examples should be ``D:\cloned\DeepSpeech\tensorflow``.
|
||||
|
||||
CPU
|
||||
~~~
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ build:
|
|||
routes: []
|
||||
maxRunTime: 3600
|
||||
docker_image: "ubuntu:16.04"
|
||||
generic:
|
||||
workerType: 'ds-macos-light'
|
||||
system_setup:
|
||||
>
|
||||
true
|
||||
|
|
@ -17,12 +19,13 @@ build:
|
|||
>
|
||||
true
|
||||
scripts:
|
||||
setup: ''
|
||||
build: ''
|
||||
package: ''
|
||||
nc_asset_name: 'native_client.tar.xz'
|
||||
args:
|
||||
tests_cmdline: ''
|
||||
tensorflow_git_desc: 'TensorFlow: v2.2.0-12-gc29895f'
|
||||
tensorflow_git_desc: 'TensorFlow: v2.2.0-14-g7ead558'
|
||||
test_model_task: ''
|
||||
homebrew:
|
||||
url: ''
|
||||
|
|
|
|||
|
|
@ -8,9 +8,17 @@ python:
|
|||
training:
|
||||
packages_xenial:
|
||||
apt: 'libopus0'
|
||||
deepspeech:
|
||||
packages_xenial:
|
||||
apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev'
|
||||
tensorflow:
|
||||
packages_xenial:
|
||||
apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python zlib1g-dev'
|
||||
apt: 'apt-get -qq update && apt-get -qq -y install realpath build-essential python-virtualenv python-dev python-pip libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip'
|
||||
packages_macos:
|
||||
brew: '$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tf_tc-brew.sh'
|
||||
packages_win:
|
||||
pacman: 'pacman --noconfirm -S patch unzip tar'
|
||||
msys64: 'ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64'
|
||||
java:
|
||||
packages_xenial:
|
||||
apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f'
|
||||
|
|
@ -132,6 +140,34 @@ system:
|
|||
win:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz"
|
||||
namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118"
|
||||
tensorflow:
|
||||
linux_amd64_cpu:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cpu/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cpu"
|
||||
linux_amd64_cuda:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cuda/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cuda"
|
||||
linux_armv7:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm"
|
||||
linux_arm64:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm64/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm64"
|
||||
darwin_amd64:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.osx/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.osx"
|
||||
android_arm64:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-arm64/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-arm64"
|
||||
android_armv7:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-armv7/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-armv7"
|
||||
win_amd64_cpu:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win"
|
||||
win_amd64_cuda:
|
||||
url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win-cuda/artifacts/public/home.tar.xz"
|
||||
namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win-cuda"
|
||||
username: 'build-user'
|
||||
homedir:
|
||||
linux: '/home/build-user'
|
||||
|
|
@ -141,7 +177,3 @@ system:
|
|||
msys2:
|
||||
url: 'https://github.com/msys2/msys2-installer/releases/download/2020-06-02/msys2-base-x86_64-20200602.tar.xz'
|
||||
sha: '598ceeaa3e2ccf86a25a2e3c449d00a9fd35300e36011bee610036dfa59d670a'
|
||||
msys2_filesystem_pkg:
|
||||
url: 'http://repo.msys2.org/msys/x86_64/filesystem-2020.02-3-x86_64.pkg.tar.xz'
|
||||
sha: '927b020a67a05139ee1b2c45bff491c1d42335e64350cc7758ee20d7c3099477'
|
||||
install: 'pacman -Udd --noconfirm $USERPROFILE/filesystem-2020.02-3-x86_64.pkg.tar.xz'
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ arm_flavor=$1
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
do_deepspeech_java_apk_build
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ source $(dirname "$0")/tc-tests-utils.sh
|
|||
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
||||
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/java/app/build/outputs/apk/release/app*.apk ${TASKCLUSTER_ARTIFACTS}/
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/java/libdeepspeech/build/outputs/aar/libdeepspeech*.aar ${TASKCLUSTER_ARTIFACTS}/
|
||||
|
|
|
|||
|
|
@ -4,12 +4,14 @@ build:
|
|||
- "swig-linux-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-linux-amd64"
|
||||
- "tf_android-arm64-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64"
|
||||
- "index.project.deepspeech.deepspeech.native_client.android-arm64.${event.head.sha}"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-arm64/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.android_arm64.url}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android-build.sh arm64-v8a"
|
||||
package: "taskcluster/android-package.sh arm64-v8a"
|
||||
nc_asset_name: "native_client.arm64.cpu.android.tar.xz"
|
||||
|
|
|
|||
|
|
@ -4,11 +4,12 @@ build:
|
|||
- "swig-linux-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-linux-amd64"
|
||||
- "tf_android-armv7-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7"
|
||||
- "index.project.deepspeech.deepspeech.native_client.android-armv7.${event.head.sha}"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-armv7/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.android_armv7.url}
|
||||
scripts:
|
||||
build: "taskcluster/android-build.sh armeabi-v7a"
|
||||
package: "taskcluster/android-package.sh armeabi-v7a"
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ arm_flavor=$1
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
BAZEL_TARGETS="
|
||||
//native_client:libdeepspeech.so
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.arm64_v8a.android_24.url}
|
||||
artifact_namespace: ${system.android_cache.arm64_v8a.android_24.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh arm64-v8a android-24"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.arm64_v8a.android_25.url}
|
||||
artifact_namespace: ${system.android_cache.arm64_v8a.android_25.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh arm64-v8a android-25"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.armeabi_v7a.android_24.url}
|
||||
artifact_namespace: ${system.android_cache.armeabi_v7a.android_24.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh armeabi-v7a android-24 default"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.armeabi_v7a.android_25.url}
|
||||
artifact_namespace: ${system.android_cache.armeabi_v7a.android_25.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh armeabi-v7a android-25"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.sdk.android_27.url}
|
||||
artifact_namespace: ${system.android_cache.sdk.android_27.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh sdk android-27"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.x86_64.android_24.url}
|
||||
artifact_namespace: ${system.android_cache.x86_64.android_24.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh x86_64 android-24"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.x86_64.android_25.url}
|
||||
artifact_namespace: ${system.android_cache.x86_64.android_25.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh x86_64 android-25"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.x86_64.android_26.url}
|
||||
artifact_namespace: ${system.android_cache.x86_64.android_26.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh x86_64 android-26"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.x86_64.android_28.url}
|
||||
artifact_namespace: ${system.android_cache.x86_64.android_28.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh x86_64 android-28"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.x86_64.android_29.url}
|
||||
artifact_namespace: ${system.android_cache.x86_64.android_29.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh x86_64 android-29"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
artifact_url: ${system.android_cache.x86_64.android_30.url}
|
||||
artifact_namespace: ${system.android_cache.x86_64.android_30.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/android_cache-build.sh x86_64 android-30"
|
||||
package: "taskcluster/android_cache-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ build:
|
|||
system_setup:
|
||||
>
|
||||
${java.packages_xenial.apt}
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-armv7/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.android_armv7.url}
|
||||
gradle_cache:
|
||||
url: ${system.gradle_cache.url}
|
||||
namespace: ${system.gradle_cache.namespace}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,6 @@ source $(dirname "$0")/tc-tests-utils.sh
|
|||
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
||||
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
|
||||
package_native_client_ndk "native_client.tar.xz" "${arm_flavor}"
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ build:
|
|||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64"
|
||||
- "index.project.deepspeech.deepspeech.native_client.android-x86_64.${event.head.sha}"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-arm64/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.android_arm64.url}
|
||||
scripts:
|
||||
build: "taskcluster/android-build.sh x86_64"
|
||||
package: "taskcluster/android-package.sh x86_64"
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -xe
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
BAZEL_TARGETS="
|
||||
//native_client:libdeepspeech.so
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -xe
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
BAZEL_TARGETS="
|
||||
//native_client:libdeepspeech.so
|
||||
|
|
|
|||
|
|
@ -5,11 +5,12 @@ build:
|
|||
- "node-gyp-cache"
|
||||
- "homebrew_builds-darwin-amd64"
|
||||
- "pyenv-darwin-amd64"
|
||||
- "tf_darwin-amd64-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx"
|
||||
- "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.darwin_amd64.url}
|
||||
scripts:
|
||||
build: "taskcluster/host-build.sh"
|
||||
package: "taskcluster/package.sh"
|
||||
|
|
|
|||
|
|
@ -5,11 +5,12 @@ build:
|
|||
- "node-gyp-cache"
|
||||
- "homebrew_builds-darwin-amd64"
|
||||
- "pyenv-darwin-amd64"
|
||||
- "tf_darwin-amd64-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc"
|
||||
- "index.project.deepspeech.deepspeech.native_client.osx-ctc.${event.head.sha}"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.darwin_amd64.url}
|
||||
maxRunTime: 14400
|
||||
scripts:
|
||||
build: 'taskcluster/decoder-build.sh'
|
||||
|
|
|
|||
|
|
@ -5,11 +5,12 @@ build:
|
|||
- "node-gyp-cache"
|
||||
- "homebrew_builds-darwin-amd64"
|
||||
- "pyenv-darwin-amd64"
|
||||
- "tf_darwin-amd64-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite"
|
||||
- "index.project.deepspeech.deepspeech.native_client.osx-tflite.${event.head.sha}"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.darwin_amd64.url}
|
||||
scripts:
|
||||
build: "taskcluster/host-build.sh tflite"
|
||||
package: "taskcluster/package.sh"
|
||||
|
|
|
|||
|
|
@ -60,9 +60,8 @@ payload:
|
|||
cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ &&
|
||||
exit $TASKCLUSTER_TASK_EXIT_CODE" 0 &&
|
||||
(pixz -d < $TASKCLUSTER_ORIG_TASKDIR/home.tar.xz | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
|
||||
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
|
||||
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
|
||||
ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
|
||||
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} &&
|
||||
git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ &&
|
||||
cd $TASKCLUSTER_TASK_DIR &&
|
||||
(mkdir pyenv-root/ && gtar -C pyenv-root/ -xf $TASKCLUSTER_ORIG_TASKDIR/pyenv.tar.gz) &&
|
||||
(mkdir homebrew-builds/ && gtar -C homebrew-builds/ -xf $TASKCLUSTER_ORIG_TASKDIR/homebrew-builds.tar.gz) &&
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -xe
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
export SYSTEM_TARGET=host-win
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
taskId: ${taskcluster.taskId}
|
||||
provisionerId: ${taskcluster.generic.provisionerId}
|
||||
workerType: ${taskcluster.generic.workerType}
|
||||
workerType: ${build.generic.workerType}
|
||||
taskGroupId: ${taskcluster.taskGroupId}
|
||||
schedulerId: ${taskcluster.schedulerId}
|
||||
dependencies:
|
||||
|
|
@ -24,6 +24,8 @@ payload:
|
|||
- "--login"
|
||||
- "-cxe"
|
||||
- $let:
|
||||
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
|
||||
extraSystemConfig: { $eval: strip(str(build.system_config)) }
|
||||
taskIndexExpire: { $fromNow: '6 months' }
|
||||
in: >
|
||||
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
|
||||
|
|
@ -32,13 +34,14 @@ payload:
|
|||
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
|
||||
export LC_ALL=C &&
|
||||
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
|
||||
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
|
||||
env &&
|
||||
mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
|
||||
cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` &&
|
||||
if [ "$cache_artifact" != "200" ]; then
|
||||
git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} &&
|
||||
cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} &&
|
||||
${extraSystemConfig} &&
|
||||
$TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.setup} &&
|
||||
$TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} &&
|
||||
$TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} &&
|
||||
$TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} 127.0.0.1:8080 ${build.cache.artifact_namespace}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ payload:
|
|||
- "-cxe"
|
||||
- $let:
|
||||
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
|
||||
extraSystemConfig: { $eval: strip(str(build.system_config)) }
|
||||
taskIndexExpire: { $fromNow: '6 months' }
|
||||
in: >
|
||||
(apt-get -qq -y remove --purge ubuntu-advantage-tools || true) &&
|
||||
|
|
@ -31,7 +32,8 @@ payload:
|
|||
adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ &&
|
||||
mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts &&
|
||||
echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${build.build_or_cache.repo} ~/${build.build_or_cache.dir}/ && cd ~/${build.build_or_cache.dir}/ && git checkout --quiet ${build.build_or_cache.sha})" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
|
||||
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh &&
|
||||
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
|
||||
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.setup} &&
|
||||
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.build} &&
|
||||
sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.package} &&
|
||||
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}
|
||||
|
|
|
|||
|
|
@ -20,10 +20,6 @@ payload:
|
|||
content:
|
||||
sha256: ${system.msys2.sha}
|
||||
url: ${system.msys2.url}
|
||||
- file: filesystem-2020.02-3-x86_64.pkg.tar.xz
|
||||
content:
|
||||
sha256: ${system.msys2_filesystem_pkg.sha}
|
||||
url: ${system.msys2_filesystem_pkg.url}
|
||||
|
||||
env:
|
||||
TC_MSYS_VERSION: 'MSYS_NT-6.3-9600'
|
||||
|
|
@ -34,18 +30,19 @@ payload:
|
|||
"C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
|
||||
"C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
|
||||
- $let:
|
||||
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
|
||||
extraSystemConfig: { $eval: strip(str(build.system_config)) }
|
||||
taskIndexExpire: { $fromNow: '6 months' }
|
||||
in: >
|
||||
echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C &&
|
||||
export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" &&
|
||||
export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" &&
|
||||
export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
|
||||
(mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) &&
|
||||
echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh &&
|
||||
env && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k
|
||||
env && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && ${extraSystemConfig} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.setup} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k
|
||||
|
||||
- .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE"
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
>
|
||||
${java.packages_xenial.apt}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/gradle-build.sh"
|
||||
package: "taskcluster/gradle-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
artifact_url: ${system.homebrew_builds.url}
|
||||
artifact_namespace: ${system.homebrew_builds.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/homebrew-build.sh --builds"
|
||||
package: "taskcluster/homebrew-package.sh --builds"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
artifact_url: ${system.homebrew_tests.url}
|
||||
artifact_namespace: ${system.homebrew_tests.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/homebrew-build.sh --tests"
|
||||
package: "taskcluster/homebrew-package.sh --tests"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ runtime=$1
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
BAZEL_TARGETS="
|
||||
//native_client:libdeepspeech.so
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
- "swig-linux-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-linux-amd64"
|
||||
- "tf_linux-amd64-cpu-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu"
|
||||
|
|
@ -12,8 +13,9 @@ build:
|
|||
>
|
||||
${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning}
|
||||
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.linux_amd64_cpu.url}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/host-build.sh"
|
||||
package: "taskcluster/package.sh"
|
||||
nc_asset_name: "native_client.amd64.cpu.linux.tar.xz"
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
- "swig-linux-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-linux-amd64"
|
||||
- "tf_linux-amd64-cpu-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-ctc"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-ctc"
|
||||
|
|
@ -12,8 +13,9 @@ build:
|
|||
>
|
||||
${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning}
|
||||
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.linux_amd64_cpu.url}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: 'taskcluster/decoder-build.sh'
|
||||
package: 'taskcluster/decoder-package.sh'
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
- "swig-linux-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-linux-amd64"
|
||||
- "tf_linux-amd64-gpu-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu"
|
||||
|
|
@ -12,9 +13,10 @@ build:
|
|||
>
|
||||
${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning}
|
||||
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.gpu/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.linux_amd64_cuda.url}
|
||||
maxRunTime: 14400
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/cuda-build.sh"
|
||||
package: "taskcluster/package.sh"
|
||||
nc_asset_name: "native_client.amd64.cuda.linux.tar.xz"
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
- "swig-linux-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-linux-amd64"
|
||||
- "tf_linux-amd64-cpu-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite"
|
||||
|
|
@ -12,8 +13,9 @@ build:
|
|||
>
|
||||
${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning}
|
||||
&& apt-get -qq update && apt-get -qq -y install nodejs python-yaml
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.linux_amd64_cpu.url}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/host-build.sh tflite"
|
||||
package: "taskcluster/package.sh"
|
||||
nc_asset_name: "native_client.amd64.tflite.linux.tar.xz"
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
- "swig-linux-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-linux-amd64"
|
||||
- "tf_linux-arm64-cpu-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64"
|
||||
|
|
@ -19,8 +20,9 @@ build:
|
|||
system_config:
|
||||
>
|
||||
multistrap -d /tmp/multistrap-armbian64-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_armbian64_buster.conf
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.arm64/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.linux_arm64.url}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/arm64-build.sh"
|
||||
package: "taskcluster/package.sh"
|
||||
nc_asset_name: "native_client.arm64.cpu.linux.tar.xz"
|
||||
|
|
|
|||
|
|
@ -44,9 +44,9 @@ then:
|
|||
extraSystemConfig: { $eval: strip(str(build.system_config)) }
|
||||
in: >
|
||||
adduser --system --home ${system.homedir.linux} ${system.username} &&
|
||||
apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} &&
|
||||
apt-get -qq update && apt-get -qq -y install ${deepspeech.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} &&
|
||||
cd ${system.homedir.linux}/ &&
|
||||
echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
|
||||
echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && cd ~/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
|
||||
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
|
||||
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
|
||||
sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
- "swig-linux-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-linux-amd64"
|
||||
- "tf_linux-rpi3-cpu-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm"
|
||||
|
|
@ -19,8 +20,9 @@ build:
|
|||
system_config:
|
||||
>
|
||||
multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.arm/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.linux_armv7.url}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/rpi3-build.sh"
|
||||
package: "taskcluster/package.sh"
|
||||
nc_asset_name: "native_client.rpi3.cpu.linux.tar.xz"
|
||||
|
|
|
|||
|
|
@ -6,6 +6,6 @@ package_option=$1
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
do_deepspeech_npm_package "${package_option}"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ build:
|
|||
>
|
||||
(apt-get -qq -y install sudo || true)
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/node-gyp-populate.sh"
|
||||
package: "taskcluster/node-gyp-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ then:
|
|||
adduser --system --home ${system.homedir.linux} ${system.username} &&
|
||||
apt-get -qq update && apt-get -qq -y install realpath git wget curl make sudo && ${extraSystemSetup} &&
|
||||
cd ${system.homedir.linux}/ &&
|
||||
echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ~/DeepSpeech/tf/ && touch ~/DeepSpeech/tf/tc-vars.sh && chmod +x ~/DeepSpeech/tf/tc-vars.sh && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
|
||||
echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
|
||||
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&
|
||||
sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} &&
|
||||
sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh
|
|||
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
||||
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
|
||||
package_native_client "native_client.tar.xz"
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
artifact_url: ${system.pyenv.osx.url}
|
||||
artifact_namespace: ${system.pyenv.osx.namespace}
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/pyenv-build.sh"
|
||||
package: "taskcluster/pyenv-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ build:
|
|||
>
|
||||
apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_xenial.apt} wget
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/pyenv-build.sh"
|
||||
package: "taskcluster/pyenv-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ build:
|
|||
artifact_url: "${system.pyenv.win.url}"
|
||||
artifact_namespace: "${system.pyenv.win.namespace}"
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/pyenv-build.sh"
|
||||
package: "taskcluster/pyenv-package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -xe
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
BAZEL_TARGETS="
|
||||
//native_client:libdeepspeech.so
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ build:
|
|||
artifact_url: "${system.swig_build.osx.url}"
|
||||
artifact_namespace: "${system.swig_build.osx.namespace}"
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/build.sh"
|
||||
package: "taskcluster/package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ build:
|
|||
>
|
||||
apt-get -qq -y install autoconf automake bison build-essential
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/build.sh"
|
||||
package: "taskcluster/package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ build:
|
|||
apt-get -qq -y install autoconf automake bison build-essential mingw-w64 &&
|
||||
(apt-get -qq -y install sudo || true)
|
||||
scripts:
|
||||
setup: ""
|
||||
build: "taskcluster/build.sh x86_64-w64-mingw32"
|
||||
package: "taskcluster/package.sh"
|
||||
metadata:
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ verify_bazel_rebuild()
|
|||
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
||||
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
|
||||
spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libdeepspeech.so' | wc -l)
|
||||
if [ "${spurious_rebuilds}" -ne 0 ]; then
|
||||
|
|
@ -108,13 +108,13 @@ verify_bazel_rebuild()
|
|||
if is_patched_bazel; then
|
||||
mkdir -p ${DS_ROOT_TASK}/DeepSpeech/ckd/ds ${DS_ROOT_TASK}/DeepSpeech/ckd/tf
|
||||
tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
|
||||
tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/
|
||||
tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/
|
||||
|
||||
echo "Making a diff between CKD files"
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS}
|
||||
diff -urNw ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff
|
||||
diff -urNw ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff
|
||||
|
||||
rm -fr ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
|
||||
rm -fr ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/
|
||||
else
|
||||
echo "Cannot get CKD information from release, please use patched Bazel"
|
||||
fi;
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ export ANDROID_TMP_DIR=/data/local/tmp
|
|||
|
||||
mkdir -p ${TASKCLUSTER_TMP_DIR} || true
|
||||
|
||||
export DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/tf
|
||||
export DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow
|
||||
export DS_DSDIR=${DS_ROOT_TASK}/DeepSpeech/ds
|
||||
export DS_EXAMPLEDIR=${DS_ROOT_TASK}/DeepSpeech/examples
|
||||
|
||||
|
|
|
|||
|
|
@ -173,26 +173,26 @@ do_deepspeech_npm_package()
|
|||
|
||||
do_bazel_build()
|
||||
{
|
||||
cd ${DS_ROOT_TASK}/DeepSpeech/tf
|
||||
cd ${DS_TFDIR}
|
||||
eval "export ${BAZEL_ENV_FLAGS}"
|
||||
|
||||
if is_patched_bazel; then
|
||||
find ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T -
|
||||
find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T -
|
||||
fi;
|
||||
|
||||
bazel ${BAZEL_OUTPUT_USER_ROOT} build \
|
||||
-s --explain bazel_monolithic.log --verbose_explanations --experimental_strict_action_env --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS}
|
||||
|
||||
if is_patched_bazel; then
|
||||
find ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T -
|
||||
find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T -
|
||||
fi;
|
||||
|
||||
verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/tf/bazel_monolithic.log"
|
||||
verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_monolithic.log"
|
||||
}
|
||||
|
||||
shutdown_bazel()
|
||||
{
|
||||
cd ${DS_ROOT_TASK}/DeepSpeech/tf
|
||||
cd ${DS_TFDIR}
|
||||
bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,238 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from glob import glob
|
||||
from functools import reduce
|
||||
|
||||
import json
|
||||
import jsone
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
import slugid
|
||||
import yaml
|
||||
import subprocess
|
||||
|
||||
import networkx as nx
|
||||
|
||||
TASKS_ROOT = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])))
|
||||
TASKCLUSTER_API_BASEURL = 'http://taskcluster/queue/v1/task/%(task_id)s'
|
||||
|
||||
def string_to_dict(sid, value):
|
||||
parts = sid.split('.')
|
||||
|
||||
def pack(parts):
|
||||
if len(parts) == 1:
|
||||
return {parts[0]: value}
|
||||
elif len(parts):
|
||||
return {parts[0]: pack(parts[1:])}
|
||||
return parts
|
||||
|
||||
return pack(parts)
|
||||
|
||||
def merge_dicts(*dicts):
|
||||
if not reduce(lambda x, y: isinstance(y, dict) and x, dicts, True):
|
||||
raise TypeError("Object in *dicts not of type dict")
|
||||
if len(dicts) < 2:
|
||||
raise ValueError("Requires 2 or more dict objects")
|
||||
|
||||
def merge(a, b):
|
||||
for d in set(a.keys()).union(b.keys()):
|
||||
if d in a and d in b:
|
||||
if type(a[d]) == type(b[d]):
|
||||
if not isinstance(a[d], dict):
|
||||
ret = list({a[d], b[d]})
|
||||
if len(ret) == 1: ret = ret[0]
|
||||
yield (d, sorted(ret))
|
||||
else:
|
||||
yield (d, dict(merge(a[d], b[d])))
|
||||
else:
|
||||
raise TypeError("Conflicting key:value type assignment", type(a[d]), a[d], type(b[d]), b[d])
|
||||
elif d in a:
|
||||
yield (d, a[d])
|
||||
elif d in b:
|
||||
yield (d, b[d])
|
||||
else:
|
||||
raise KeyError
|
||||
|
||||
return reduce(lambda x, y: dict(merge(x, y)), dicts[1:], dicts[0])
|
||||
|
||||
def taskcluster_event_context():
|
||||
das_context = {}
|
||||
|
||||
# Pre-filterting
|
||||
for k in os.environ.keys():
|
||||
if k == 'GITHUB_HEAD_USER':
|
||||
os.environ['GITHUB_HEAD_USER_LOGIN'] = os.environ[k]
|
||||
del os.environ['GITHUB_HEAD_USER']
|
||||
|
||||
for k in os.environ.keys():
|
||||
if k == 'TASK_ID':
|
||||
parts = string_to_dict('taskcluster.taskGroupId', os.environ[k])
|
||||
das_context = merge_dicts(das_context, parts)
|
||||
|
||||
if k.startswith('GITHUB_'):
|
||||
parts = string_to_dict(k.lower().replace('_', '.').replace('github', 'event'), os.environ[k])
|
||||
das_context = merge_dicts(das_context, parts)
|
||||
|
||||
return das_context
|
||||
|
||||
def load_specific_contextFile(file):
|
||||
specific_context = {}
|
||||
|
||||
try:
|
||||
with open(os.path.join(TASKS_ROOT, file)) as src:
|
||||
specific_context = yaml.load(src)
|
||||
|
||||
if specific_context is None:
|
||||
specific_context = {}
|
||||
except FileNotFoundError:
|
||||
specific_context = {}
|
||||
|
||||
return specific_context
|
||||
|
||||
def defaultValues_build_context():
|
||||
return load_specific_contextFile('.build.yml')
|
||||
|
||||
def shared_context():
|
||||
return load_specific_contextFile('.shared.yml')
|
||||
|
||||
def create_task_payload(build, base_context):
|
||||
print('build', build)
|
||||
build_type = os.path.splitext(os.path.basename(build))[0]
|
||||
|
||||
build_context = defaultValues_build_context()
|
||||
with open(build) as src:
|
||||
build_context['build'].update(yaml.load(src)['build'])
|
||||
|
||||
# Be able to use what has been defined in base_context
|
||||
# e.g., the {${event.head.branch}}
|
||||
build_context = jsone.render(build_context, base_context)
|
||||
template_context = {
|
||||
'taskcluster': {
|
||||
'taskId': as_slugid(build_type)
|
||||
},
|
||||
'build_type': build_type
|
||||
}
|
||||
|
||||
with open(os.path.join(TASKS_ROOT, build_context['build']['template_file'])) as src:
|
||||
template = yaml.load(src)
|
||||
|
||||
contextes = merge_dicts({}, base_context, template_context, build_context)
|
||||
for one_context in glob(os.path.join(TASKS_ROOT, '*.cyml')):
|
||||
with open(one_context) as src:
|
||||
contextes = merge_dicts(contextes, yaml.load(src))
|
||||
|
||||
return jsone.render(template, contextes)
|
||||
|
||||
def send_task(t):
|
||||
url = TASKCLUSTER_API_BASEURL % {'task_id': t['taskId']}
|
||||
del t['taskId']
|
||||
|
||||
r = requests.put(url, json=t)
|
||||
|
||||
print(url, r.status_code)
|
||||
if r.status_code != requests.codes.ok:
|
||||
print(json.dumps(t, indent=2))
|
||||
print(r.content)
|
||||
print(json.loads(r.content.decode())['message'])
|
||||
|
||||
return r.status_code == requests.codes.ok
|
||||
|
||||
slugids = {}
|
||||
def as_slugid(name):
|
||||
if name not in slugids:
|
||||
slugids[name] = slugid.nice().decode()
|
||||
print('cache miss', name, slugids[name])
|
||||
else:
|
||||
print('cache hit', name, slugids[name])
|
||||
return slugids[name]
|
||||
|
||||
def to_int(x):
|
||||
return int(x)
|
||||
|
||||
def functions_context():
|
||||
return {
|
||||
'as_slugid': as_slugid,
|
||||
'to_int': to_int
|
||||
}
|
||||
|
||||
def is_dry_run():
|
||||
return (len(sys.argv) > 1) and (sys.argv[1] == '--dry')
|
||||
|
||||
def should_run():
|
||||
# Make a quick clone to fetch the last commit
|
||||
try:
|
||||
subprocess.check_call([
|
||||
'git', 'clone', '--quiet', '-b', os.environ.get('GITHUB_HEAD_BRANCH'),
|
||||
'--single-branch', os.environ.get('GITHUB_HEAD_REPO_URL'),
|
||||
'--depth=1', '/tmp/ds-clone/'
|
||||
], env={'GIT_LFS_SKIP_SMUDGE': '1'})
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("Error while git cloning:", e, file=sys.stderr)
|
||||
return False
|
||||
|
||||
try:
|
||||
git_msg = subprocess.check_output([
|
||||
'git', '--git-dir=/tmp/ds-clone/.git/',
|
||||
'log', '--format=%b', '-n', '1',
|
||||
os.environ.get('GITHUB_HEAD_SHA')
|
||||
]).decode('utf-8').strip().upper()
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("Error while git show:", e, file=sys.stderr)
|
||||
return False
|
||||
|
||||
print('Commit message:', git_msg)
|
||||
|
||||
x_deepspeech = filter(lambda x: 'X-DEEPSPEECH:' in x, git_msg.split('\n'))
|
||||
if len(list(filter(lambda x: 'NOBUILD' in x, x_deepspeech))) == 1:
|
||||
print('Not running anything according to commit message')
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not is_dry_run():
|
||||
# We might want to NOT run in some cases
|
||||
if not should_run():
|
||||
sys.exit(0)
|
||||
|
||||
base_context = taskcluster_event_context()
|
||||
base_context = merge_dicts(base_context, functions_context())
|
||||
base_context = merge_dicts(base_context, shared_context())
|
||||
|
||||
root_task = base_context['taskcluster']['taskGroupId']
|
||||
|
||||
tasks_graph = nx.DiGraph()
|
||||
tasks = {}
|
||||
|
||||
for build in glob(os.path.join(TASKS_ROOT, '*.yml')):
|
||||
t = create_task_payload(build, base_context)
|
||||
|
||||
# We allow template to produce completely empty output
|
||||
if not t:
|
||||
continue
|
||||
|
||||
if 'dependencies' in t and len(t['dependencies']) > 0:
|
||||
for dep in t['dependencies']:
|
||||
tasks_graph.add_edge(t['taskId'], dep)
|
||||
else:
|
||||
tasks_graph.add_edge(t['taskId'], root_task)
|
||||
|
||||
tasks[t['taskId']] = t
|
||||
|
||||
for task in nx.dfs_postorder_nodes(tasks_graph):
|
||||
# root_task is the task group and also the task id that is already
|
||||
# running, so we don't have to schedule that
|
||||
if task == root_task:
|
||||
continue
|
||||
|
||||
t = tasks[task]
|
||||
if is_dry_run():
|
||||
print(json.dumps(t, indent=2))
|
||||
continue
|
||||
|
||||
p = send_task(t)
|
||||
if not p:
|
||||
sys.exit(1)
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
json-e == 2.3.1
|
||||
networkx
|
||||
pyaml
|
||||
requests
|
||||
slugid == 1.0.7
|
||||
|
|
@ -2,15 +2,13 @@
|
|||
|
||||
set -ex
|
||||
|
||||
# tc-decision.py assumes being at the root folder
|
||||
curdir=$(dirname "$0")/..
|
||||
curdir=$(dirname "$0")/
|
||||
|
||||
pip3 install --quiet --user --upgrade pip
|
||||
|
||||
export PATH=$HOME/.local/bin/:$PATH
|
||||
|
||||
curl -L --silent https://raw.githubusercontent.com/lissyx/taskcluster-github-decision/${TC_DECISION_SHA}/requirements.txt | pip3 install --quiet --user --upgrade -r /dev/stdin
|
||||
curl -L --silent https://raw.githubusercontent.com/lissyx/taskcluster-github-decision/${TC_DECISION_SHA}/tc-decision.py > ${curdir}/tc-decision.py
|
||||
pip3 install --quiet --user --upgrade -r ${curdir}/tc-decision_reqs.txt
|
||||
|
||||
# First, perform dry run for push and pull request
|
||||
# This should help us track merge failures in advance
|
||||
|
|
|
|||
|
|
@ -38,8 +38,6 @@ then:
|
|||
"C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
|
||||
"C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
|
||||
- $let:
|
||||
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
|
||||
|
|
@ -69,10 +67,6 @@ then:
|
|||
content:
|
||||
sha256: ${system.msys2.sha}
|
||||
url: ${system.msys2.url}
|
||||
- file: filesystem-2020.02-3-x86_64.pkg.tar.xz
|
||||
content:
|
||||
sha256: ${system.msys2_filesystem_pkg.sha}
|
||||
url: ${system.msys2_filesystem_pkg.url}
|
||||
- file: pyenv.tar.gz
|
||||
content:
|
||||
url: ${system.pyenv.win.url}
|
||||
|
|
|
|||
|
|
@ -40,8 +40,6 @@ then:
|
|||
"C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
|
||||
"C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
|
||||
- $let:
|
||||
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
|
||||
|
|
@ -71,10 +69,6 @@ then:
|
|||
content:
|
||||
sha256: ${system.msys2.sha}
|
||||
url: ${system.msys2.url}
|
||||
- file: filesystem-2020.02-3-x86_64.pkg.tar.xz
|
||||
content:
|
||||
sha256: ${system.msys2_filesystem_pkg.sha}
|
||||
url: ${system.msys2_filesystem_pkg.url}
|
||||
- file: pyenv.tar.gz
|
||||
content:
|
||||
url: ${system.pyenv.win.url}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-linux-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.android_arm64.url}
|
||||
artifact_namespace: ${system.tensorflow.android_arm64.namespace}
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh --android"
|
||||
build: "taskcluster/tf_tc-build.sh --android-arm64"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 14400
|
||||
metadata:
|
||||
name: "TensorFlow Android ARM64"
|
||||
description: "Building TensorFlow for Android ARM64, optimized version"
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-linux-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.android_armv7.url}
|
||||
artifact_namespace: ${system.tensorflow.android_armv7.namespace}
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh --android"
|
||||
build: "taskcluster/tf_tc-build.sh --android-armv7"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 14400
|
||||
metadata:
|
||||
name: "TensorFlow Android ARMv7"
|
||||
description: "Building TensorFlow for Android ARMv7, optimized version"
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-darwin-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.darwin_amd64.url}
|
||||
artifact_namespace: ${system.tensorflow.darwin_amd64.namespace}
|
||||
generic:
|
||||
workerType: "ds-macos-heavy"
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_macos.brew}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh"
|
||||
build: "taskcluster/tf_tc-build.sh --osx"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 28800
|
||||
metadata:
|
||||
name: "TensorFlow OSX AMD64 CPU"
|
||||
description: "Building TensorFlow for OSX AMD64, CPU only, optimized version"
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-linux-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.linux_amd64_cpu.url}
|
||||
artifact_namespace: ${system.tensorflow.linux_amd64_cpu.namespace}
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh"
|
||||
build: "taskcluster/tf_tc-build.sh"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 14400
|
||||
metadata:
|
||||
name: "TensorFlow Linux AMD64 CPU"
|
||||
description: "Building TensorFlow for Linux/AMD64, CPU only, optimized version"
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-linux-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.linux_amd64_cuda.url}
|
||||
artifact_namespace: ${system.tensorflow.linux_amd64_cuda.namespace}
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh --cuda"
|
||||
build: "taskcluster/tf_tc-build.sh --gpu"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 14400
|
||||
metadata:
|
||||
name: "TensorFlow Linux AMD64 CUDA"
|
||||
description: "Building TensorFlow for Linux/AMD64, CUDA-enabled, optimized version"
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-linux-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.linux_arm64.url}
|
||||
artifact_namespace: ${system.tensorflow.linux_arm64.namespace}
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh"
|
||||
build: "taskcluster/tf_tc-build.sh --arm64"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 14400
|
||||
metadata:
|
||||
name: "TensorFlow Linux ARM64 Cortex-A53 CPU"
|
||||
description: "Building TensorFlow for Linux ARM64 Cortex-A53, CPU only, optimized version"
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-linux-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.linux_armv7.url}
|
||||
artifact_namespace: ${system.tensorflow.linux_armv7.namespace}
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh"
|
||||
build: "taskcluster/tf_tc-build.sh --arm"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 14400
|
||||
metadata:
|
||||
name: "TensorFlow Linux RPi3/ARMv7 CPU"
|
||||
description: "Building TensorFlow for Linux RPi3 ARMv7, CPU only, optimized version"
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then
|
||||
echo "No TASKCLUSTER_TASK_DIR, aborting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LOCAL_BREW="${TASKCLUSTER_TASK_DIR}/homebrew"
|
||||
export PATH=${LOCAL_BREW}/bin:$PATH
|
||||
export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/homebrew.logs/"
|
||||
export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/homebrew.cache/"
|
||||
|
||||
# Never fail on pre-existing homebrew/ directory
|
||||
mkdir -p "${LOCAL_BREW}" || true
|
||||
mkdir -p "${HOMEBREW_CACHE}" || true
|
||||
|
||||
# Make sure to verify there is a 'brew' binary there, otherwise install things.
|
||||
if [ ! -x "${LOCAL_BREW}/bin/brew" ]; then
|
||||
curl -L https://github.com/Homebrew/brew/tarball/2.2.17 | tar xz --strip 1 -C "${LOCAL_BREW}"
|
||||
fi;
|
||||
|
||||
echo "local brew list (should be empty) ..."
|
||||
brew list
|
||||
|
||||
echo "local brew prefix ..."
|
||||
local_prefix=$(brew --prefix)
|
||||
echo "${local_prefix}"
|
||||
|
||||
if [ "${LOCAL_BREW}" != "${local_prefix}" ]; then
|
||||
echo "Weird state:"
|
||||
echo "LOCAL_BREW=${LOCAL_BREW}"
|
||||
echo "local_prefix=${local_prefix}"
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
# coreutils, pyenv-virtualenv required for build of tensorflow
|
||||
all_pkgs="coreutils pyenv-virtualenv"
|
||||
|
||||
for pkg in ${all_pkgs};
|
||||
do
|
||||
(brew list --versions ${pkg} && brew upgrade ${pkg}) || brew install ${pkg}
|
||||
done;
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
source $(dirname $0)/tf_tc-vars.sh
|
||||
|
||||
build_amd64=yes
|
||||
build_gpu=no
|
||||
build_android_arm=no
|
||||
build_android_arm64=no
|
||||
build_linux_arm=no
|
||||
build_linux_arm64=no
|
||||
|
||||
if [ "$1" = "--gpu" ]; then
|
||||
build_amd64=yes
|
||||
build_gpu=yes
|
||||
build_android_arm=no
|
||||
build_android_arm64=no
|
||||
build_linux_arm=no
|
||||
build_linux_arm64=no
|
||||
fi
|
||||
|
||||
if [ "$1" = "--arm" ]; then
|
||||
build_amd64=yes
|
||||
build_gpu=no
|
||||
build_android_arm=no
|
||||
build_android_arm64=no
|
||||
build_linux_arm=yes
|
||||
build_linux_arm64=no
|
||||
fi
|
||||
|
||||
if [ "$1" = "--arm64" ]; then
|
||||
build_amd64=yes
|
||||
build_gpu=no
|
||||
build_android_arm=no
|
||||
build_android_arm64=no
|
||||
build_linux_arm=no
|
||||
build_linux_arm64=yes
|
||||
fi
|
||||
|
||||
if [ "$1" = "--android-armv7" ]; then
|
||||
build_amd64=no
|
||||
build_gpu=no
|
||||
build_android_arm=yes
|
||||
build_android_arm64=no
|
||||
build_linux_arm=no
|
||||
build_linux_arm64=no
|
||||
fi
|
||||
|
||||
if [ "$1" = "--android-arm64" ]; then
|
||||
build_amd64=no
|
||||
build_gpu=no
|
||||
build_android_arm=no
|
||||
build_android_arm64=yes
|
||||
build_linux_arm=no
|
||||
build_linux_arm64=no
|
||||
fi
|
||||
|
||||
pushd ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/
|
||||
BAZEL_BUILD="bazel ${BAZEL_OUTPUT_USER_ROOT} build -s --explain bazel_monolithic_tf.log --verbose_explanations --experimental_strict_action_env --config=monolithic"
|
||||
|
||||
# Start a bazel process to ensure reliability on Windows and avoid:
|
||||
# FATAL: corrupt installation: file 'c:\builds\tc-workdir\.bazel_cache/install/6b1660721930e9d5f231f7d2a626209b/_embedded_binaries/build-runfiles.exe' missing.
|
||||
bazel ${BAZEL_OUTPUT_USER_ROOT} info
|
||||
|
||||
# Force toolchain sync (useful on macOS ?)
|
||||
bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure
|
||||
|
||||
if [ "${build_amd64}" = "yes" ]; then
|
||||
# Pure amd64 CPU-only build
|
||||
if [ "${OS}" = "${TC_MSYS_VERSION}" -a "${build_gpu}" = "no" ]; then
|
||||
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
|
||||
elif [ "${build_gpu}" = "no" -a "${build_linux_arm}" = "no" -a "${build_linux_arm64}" = "no" ]; then
|
||||
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB}
|
||||
fi
|
||||
|
||||
# Cross RPi3 CPU-only build
|
||||
if [ "${build_linux_arm}" = "yes" ]; then
|
||||
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
|
||||
fi
|
||||
|
||||
# Cross ARM64 Cortex-A53 build
|
||||
if [ "${build_linux_arm64}" = "yes" ]; then
|
||||
echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
|
||||
fi
|
||||
|
||||
# Pure amd64 GPU-enabled build
|
||||
if [ "${build_gpu}" = "yes" ]; then
|
||||
eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} -c opt ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API}
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${build_android_arm}" = "yes" ]; then
|
||||
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
|
||||
fi;
|
||||
|
||||
if [ "${build_android_arm64}" = "yes" ]; then
|
||||
echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB}
|
||||
fi;
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
# There was a failure, just account for it.
|
||||
echo "Build failure, please check the output above. Exit code was: $?"
|
||||
return 1
|
||||
fi
|
||||
|
||||
bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown
|
||||
popd
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -xe
|
||||
|
||||
source $(dirname $0)/tf_tc-vars.sh
|
||||
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
||||
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_*.log ${TASKCLUSTER_ARTIFACTS}
|
||||
|
||||
OUTPUT_ROOT="${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin"
|
||||
|
||||
for output_bin in \
|
||||
tensorflow/lite/experimental/c/libtensorflowlite_c.so \
|
||||
tensorflow/tools/graph_transforms/transform_graph \
|
||||
tensorflow/tools/graph_transforms/summarize_graph \
|
||||
tensorflow/tools/benchmark/benchmark_model \
|
||||
tensorflow/contrib/util/convert_graphdef_memmapped_format \
|
||||
tensorflow/lite/toco/toco;
|
||||
do
|
||||
if [ -f "${OUTPUT_ROOT}/${output_bin}" ]; then
|
||||
cp ${OUTPUT_ROOT}/${output_bin} ${TASKCLUSTER_ARTIFACTS}/
|
||||
fi;
|
||||
done;
|
||||
|
||||
if [ -f "${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model" ]; then
|
||||
cp ${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model ${TASKCLUSTER_ARTIFACTS}/lite_benchmark_model
|
||||
fi
|
||||
|
||||
# It seems that bsdtar and gnutar are behaving a bit differently on the way
|
||||
# they deal with --exclude="./public/*" ; this caused ./DeepSpeech/tensorflow/core/public/
|
||||
# to be ditched when we just wanted to get rid of ./public/ on OSX.
|
||||
# Switching to gnutar (already needed for the --transform on DeepSpeech tasks)
|
||||
# does the trick.
|
||||
TAR_EXCLUDE="--exclude=./dls/*"
|
||||
if [ "${OS}" = "Darwin" ]; then
|
||||
TAR_EXCLUDE="--exclude=./dls/* --exclude=./public/* --exclude=./generic-worker/* --exclude=./homebrew/* --exclude=./homebrew.cache/* --exclude=./homebrew.logs/*"
|
||||
fi;
|
||||
|
||||
# Make a tar of
|
||||
# - /home/build-user/ (linux
|
||||
# - /Users/build-user/TaskCluster/HeavyTasks/X/ (OSX)
|
||||
# - C:\builds\tc-workdir\ (windows)
|
||||
|
||||
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
export PATH=$PATH:'/c/Program Files/7-Zip/'
|
||||
pushd ${DS_ROOT_TASK}
|
||||
7z a '-xr!.\dls\' '-xr!.\tmp\' '-xr!.\msys64\' -snl -snh -so home.tar . | 7z a -si ${TASKCLUSTER_ARTIFACTS}/home.tar.xz
|
||||
popd
|
||||
else
|
||||
${TAR} -C ${DS_ROOT_TASK} ${TAR_EXCLUDE} -cf - . | ${XZ} > ${TASKCLUSTER_ARTIFACTS}/home.tar.xz
|
||||
fi
|
||||
|
||||
if [ "${OS}" = "Linux" ]; then
|
||||
SHA_SUM_GEN="sha256sum"
|
||||
elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
SHA_SUM_GEN="sha256sum"
|
||||
elif [ "${OS}" = "Darwin" ]; then
|
||||
SHA_SUM_GEN="shasum -a 256"
|
||||
fi;
|
||||
|
||||
${SHA_SUM_GEN} ${TASKCLUSTER_ARTIFACTS}/* > ${TASKCLUSTER_ARTIFACTS}/checksums.txt
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
# Taken from https://www.tensorflow.org/install/source
|
||||
# Only future is needed for our builds, as we don't build the Python package
|
||||
pip install -U --user future==0.17.1
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
source $(dirname $0)/tf_tc-vars.sh
|
||||
|
||||
install_cuda=
|
||||
if [ "$1" = "--cuda" ]; then
|
||||
install_cuda=yes
|
||||
fi
|
||||
|
||||
install_android=
|
||||
if [ "$1" = "--android" ]; then
|
||||
install_android=yes
|
||||
fi
|
||||
|
||||
# $1 url
|
||||
# $2 sha256
|
||||
download()
|
||||
{
|
||||
fname=`basename $1`
|
||||
|
||||
${WGET} $1 -O ${DS_ROOT_TASK}/dls/$fname && echo "$2 ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} -
|
||||
}
|
||||
|
||||
# Download stuff
|
||||
mkdir -p ${DS_ROOT_TASK}/dls || true
|
||||
download $BAZEL_URL $BAZEL_SHA256
|
||||
|
||||
if [ ! -z "${install_cuda}" ]; then
|
||||
download $CUDA_URL $CUDA_SHA256
|
||||
download $CUDNN_URL $CUDNN_SHA256
|
||||
fi;
|
||||
|
||||
if [ ! -z "${install_android}" ]; then
|
||||
download $ANDROID_NDK_URL $ANDROID_NDK_SHA256
|
||||
download $ANDROID_SDK_URL $ANDROID_SDK_SHA256
|
||||
fi;
|
||||
|
||||
# For debug
|
||||
ls -hal ${DS_ROOT_TASK}/dls/
|
||||
|
||||
# Install Bazel in ${DS_ROOT_TASK}/bin
|
||||
BAZEL_INSTALL_FILENAME=$(basename "${BAZEL_URL}")
|
||||
if [ "${OS}" = "Linux" ]; then
|
||||
BAZEL_INSTALL_FLAGS="--user"
|
||||
elif [ "${OS}" = "Darwin" ]; then
|
||||
BAZEL_INSTALL_FLAGS="--bin=${DS_ROOT_TASK}/bin --base=${DS_ROOT_TASK}/.bazel"
|
||||
fi;
|
||||
mkdir -p ${DS_ROOT_TASK}/bin || true
|
||||
pushd ${DS_ROOT_TASK}/bin
|
||||
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel.exe
|
||||
else
|
||||
/bin/bash ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${BAZEL_INSTALL_FLAGS}
|
||||
fi
|
||||
popd
|
||||
|
||||
# For debug
|
||||
bazel version
|
||||
|
||||
bazel shutdown
|
||||
|
||||
if [ ! -z "${install_cuda}" ]; then
|
||||
# Install CUDA and CuDNN
|
||||
mkdir -p ${DS_ROOT_TASK}/DeepSpeech/CUDA/ || true
|
||||
pushd ${DS_ROOT_TASK}
|
||||
CUDA_FILE=`basename ${CUDA_URL}`
|
||||
PERL5LIB=. sh ${DS_ROOT_TASK}/dls/${CUDA_FILE} --silent --override --toolkit --toolkitpath=${DS_ROOT_TASK}/DeepSpeech/CUDA/ --defaultroot=${DS_ROOT_TASK}/DeepSpeech/CUDA/
|
||||
|
||||
CUDNN_FILE=`basename ${CUDNN_URL}`
|
||||
tar xvf ${DS_ROOT_TASK}/dls/${CUDNN_FILE} --strip-components=1 -C ${DS_ROOT_TASK}/DeepSpeech/CUDA/
|
||||
popd
|
||||
|
||||
LD_LIBRARY_PATH=${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/:${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/:$LD_LIBRARY_PATH
|
||||
export LD_LIBRARY_PATH
|
||||
|
||||
# We might lack libcuda.so.1 symlink, let's fix as upstream does:
|
||||
# https://github.com/tensorflow/tensorflow/pull/13811/files?diff=split#diff-2352449eb75e66016e97a591d3f0f43dR96
|
||||
if [ ! -h "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so.1" ]; then
|
||||
ln -s "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so" "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so.1"
|
||||
fi;
|
||||
|
||||
else
|
||||
echo "No CUDA/CuDNN to install"
|
||||
fi
|
||||
|
||||
if [ ! -z "${install_android}" ]; then
|
||||
mkdir -p ${DS_ROOT_TASK}/DeepSpeech/Android/SDK || true
|
||||
ANDROID_NDK_FILE=`basename ${ANDROID_NDK_URL}`
|
||||
ANDROID_SDK_FILE=`basename ${ANDROID_SDK_URL}`
|
||||
|
||||
pushd ${DS_ROOT_TASK}/DeepSpeech/Android
|
||||
unzip ${DS_ROOT_TASK}/dls/${ANDROID_NDK_FILE}
|
||||
popd
|
||||
|
||||
pushd ${DS_ROOT_TASK}/DeepSpeech/Android/SDK
|
||||
unzip ${DS_ROOT_TASK}/dls/${ANDROID_SDK_FILE}
|
||||
yes | ./tools/bin/sdkmanager --licenses
|
||||
./tools/bin/sdkmanager --update
|
||||
./tools/bin/sdkmanager --install "platforms;android-16" "build-tools;28.0.3"
|
||||
popd
|
||||
fi
|
||||
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
||||
|
||||
pushd ${DS_ROOT_TASK}/DeepSpeech/ds/
|
||||
git submodule --quiet sync tensorflow/ && git submodule --quiet update --init tensorflow/
|
||||
popd
|
||||
|
||||
|
||||
# Taken from https://www.tensorflow.org/install/source
|
||||
# Only future is needed for our builds, as we don't build the Python package
|
||||
pip install -U --user future==0.17.1 || true
|
||||
|
|
@ -0,0 +1,195 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
export OS=$(uname)
|
||||
if [ "${OS}" = "Linux" ]; then
|
||||
export DS_ROOT_TASK=$(/usr/bin/realpath "${HOME}")
|
||||
|
||||
BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-linux-x86_64.sh
|
||||
BAZEL_SHA256=2fbdc9c0e3d376697caf0ee3673b7c9475214068c55a01b9744891e131f90b87
|
||||
|
||||
CUDA_URL=http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run
|
||||
CUDA_SHA256=e7c22dc21278eb1b82f34a60ad7640b41ad3943d929bebda3008b72536855d31
|
||||
|
||||
# From https://gitlab.com/nvidia/cuda/blob/centos7/10.1/devel/cudnn7/Dockerfile
|
||||
CUDNN_URL=http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.0/cudnn-10.1-linux-x64-v7.6.0.64.tgz
|
||||
CUDNN_SHA256=e956c6f9222fcb867a10449cfc76dee5cfd7c7531021d95fe9586d7e043b57d7
|
||||
|
||||
ANDROID_NDK_URL=https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip
|
||||
ANDROID_NDK_SHA256=4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd
|
||||
|
||||
ANDROID_SDK_URL=https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip
|
||||
ANDROID_SDK_SHA256=92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9
|
||||
|
||||
SHA_SUM="sha256sum -c --strict"
|
||||
WGET=/usr/bin/wget
|
||||
TAR=tar
|
||||
XZ="pixz -9"
|
||||
elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
if [ -z "${TASKCLUSTER_TASK_DIR}" -o -z "${TASKCLUSTER_ARTIFACTS}" ]; then
|
||||
echo "Inconsistent Windows setup: missing some vars."
|
||||
echo "TASKCLUSTER_TASK_DIR=${TASKCLUSTER_TASK_DIR}"
|
||||
echo "TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS}"
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
# Re-export with cygpath to make sure it is sane, otherwise it might trigger
|
||||
# unobvious failures with cp etc.
|
||||
export TASKCLUSTER_TASK_DIR="$(cygpath ${TASKCLUSTER_TASK_DIR})"
|
||||
export TASKCLUSTER_ARTIFACTS="$(cygpath ${TASKCLUSTER_ARTIFACTS})"
|
||||
|
||||
export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
|
||||
export BAZEL_VC='C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC'
|
||||
export BAZEL_SH='C:\builds\tc-workdir\msys64\usr\bin\bash'
|
||||
export TC_WIN_BUILD_PATH='C:\builds\tc-workdir\msys64\usr\bin;C:\Python36'
|
||||
export MSYS2_ARG_CONV_EXCL='//'
|
||||
|
||||
mkdir -p ${TASKCLUSTER_TASK_DIR}/tmp/
|
||||
export TEMP=${TASKCLUSTER_TASK_DIR}/tmp/
|
||||
export TMP=${TASKCLUSTER_TASK_DIR}/tmp/
|
||||
|
||||
BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-windows-x86_64.exe
|
||||
BAZEL_SHA256=cc7b3ff6f4bfd6bc2121a80656afec66ee57713e8b88e9d2fb58b4eddf271268
|
||||
|
||||
CUDA_INSTALL_DIRECTORY=$(cygpath 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1')
|
||||
|
||||
SHA_SUM="sha256sum -c --strict"
|
||||
WGET=wget
|
||||
TAR=/usr/bin/tar.exe
|
||||
XZ="xz -9 -T0"
|
||||
elif [ "${OS}" = "Darwin" ]; then
|
||||
if [ -z "${TASKCLUSTER_TASK_DIR}" -o -z "${TASKCLUSTER_ARTIFACTS}" ]; then
|
||||
echo "Inconsistent OSX setup: missing some vars."
|
||||
echo "TASKCLUSTER_TASK_DIR=${TASKCLUSTER_TASK_DIR}"
|
||||
echo "TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS}"
|
||||
exit 1
|
||||
fi;
|
||||
|
||||
export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
|
||||
|
||||
BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-darwin-x86_64.sh
|
||||
BAZEL_SHA256=c675fa27d99a3114d681db10eb03ded547c40f702b2048c99b8f4ea8e89b9356
|
||||
|
||||
SHA_SUM="shasum -a 256 -c"
|
||||
WGET=wget
|
||||
TAR=gtar
|
||||
XZ="pixz -9"
|
||||
fi;
|
||||
|
||||
# /tmp/artifacts for docker-worker on linux,
|
||||
# and task subdir for generic-worker on osx
|
||||
export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts}
|
||||
|
||||
### Define variables that needs to be exported to other processes
|
||||
|
||||
PATH=${DS_ROOT_TASK}/bin:$PATH
|
||||
if [ "${OS}" = "Darwin" ]; then
|
||||
PATH=${DS_ROOT_TASK}/homebrew/bin/:${DS_ROOT_TASK}/homebrew/opt/node@10/bin:$PATH
|
||||
fi;
|
||||
export PATH
|
||||
|
||||
if [ "${OS}" = "Linux" ]; then
|
||||
export LD_LIBRARY_PATH=${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/:${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/:$LD_LIBRARY_PATH
|
||||
export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/
|
||||
export ANDROID_NDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/android-ndk-r18b/
|
||||
fi;
|
||||
|
||||
export TF_ENABLE_XLA=0
|
||||
if [ "${OS}" = "Linux" ]; then
|
||||
TF_NEED_JEMALLOC=1
|
||||
elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
TF_NEED_JEMALLOC=0
|
||||
elif [ "${OS}" = "Darwin" ]; then
|
||||
TF_NEED_JEMALLOC=0
|
||||
fi;
|
||||
export TF_NEED_JEMALLOC
|
||||
export TF_NEED_OPENCL_SYCL=0
|
||||
export TF_NEED_MKL=0
|
||||
export TF_NEED_VERBS=0
|
||||
export TF_NEED_MPI=0
|
||||
export TF_NEED_IGNITE=0
|
||||
export TF_NEED_GDR=0
|
||||
export TF_NEED_NGRAPH=0
|
||||
export TF_DOWNLOAD_CLANG=0
|
||||
export TF_SET_ANDROID_WORKSPACE=0
|
||||
export TF_NEED_TENSORRT=0
|
||||
export TF_NEED_ROCM=0
|
||||
|
||||
# This should be gcc-5, hopefully. CUDA and TensorFlow might not be happy, otherwise.
|
||||
export GCC_HOST_COMPILER_PATH=/usr/bin/gcc
|
||||
|
||||
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
export PYTHON_BIN_PATH=C:/Python36/python.exe
|
||||
else
|
||||
export PYTHON_BIN_PATH=/usr/bin/python2.7
|
||||
fi
|
||||
|
||||
## Below, define or export some build variables
|
||||
|
||||
# Enable some SIMD support. Limit ourselves to what Tensorflow needs.
|
||||
# Also ensure to not require too recent CPU: AVX2/FMA introduced by:
|
||||
# - Intel with Haswell (2013)
|
||||
# - AMD with Excavator (2015)
|
||||
# For better compatibility, AVX ony might be better.
|
||||
#
|
||||
# Build for generic amd64 platforms, no device-specific optimization
|
||||
# See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for targetting specific CPUs
|
||||
|
||||
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
CC_OPT_FLAGS="/arch:AVX"
|
||||
else
|
||||
CC_OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx"
|
||||
fi
|
||||
BAZEL_OPT_FLAGS=""
|
||||
for flag in ${CC_OPT_FLAGS};
|
||||
do
|
||||
BAZEL_OPT_FLAGS="${BAZEL_OPT_FLAGS} --copt=${flag}"
|
||||
done;
|
||||
|
||||
export CC_OPT_FLAGS
|
||||
|
||||
BAZEL_OUTPUT_CACHE_DIR="${DS_ROOT_TASK}/.bazel_cache/"
|
||||
BAZEL_OUTPUT_CACHE_INSTANCE="${BAZEL_OUTPUT_CACHE_DIR}/output/"
|
||||
mkdir -p ${BAZEL_OUTPUT_CACHE_INSTANCE} || true
|
||||
|
||||
# We need both to ensure stable path ; default value for output_base is some
|
||||
# MD5 value.
|
||||
BAZEL_OUTPUT_USER_ROOT="--output_user_root ${BAZEL_OUTPUT_CACHE_DIR} --output_base ${BAZEL_OUTPUT_CACHE_INSTANCE}"
|
||||
export BAZEL_OUTPUT_USER_ROOT
|
||||
|
||||
NVCC_COMPUTE="3.5"
|
||||
|
||||
### Define build parameters/env variables that we will re-ues in sourcing scripts.
|
||||
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_PATHS=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\""
|
||||
else
|
||||
TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${DS_ROOT_TASK}/DeepSpeech/CUDA\" TF_CUDA_PATHS=\"${DS_ROOT_TASK}/DeepSpeech/CUDA\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\""
|
||||
fi
|
||||
BAZEL_ARM_FLAGS="--config=rpi3 --config=rpi3_opt --copt=-DTFLITE_WITH_RUY_GEMV"
|
||||
BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_WITH_RUY_GEMV"
|
||||
BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV"
|
||||
BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV"
|
||||
BAZEL_CUDA_FLAGS="--config=cuda"
|
||||
|
||||
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
|
||||
# Somehow, even with Python being in the PATH, Bazel on windows struggles
|
||||
# with '/usr/bin/env python' ...
|
||||
#
|
||||
# We also force TMP/TEMP otherwise Bazel will pick default Windows one
|
||||
# under %USERPROFILE%\AppData\Local\Temp and with 8.3 file format convention
|
||||
# it messes with cxx_builtin_include_directory
|
||||
BAZEL_EXTRA_FLAGS="--action_env=PATH=${TC_WIN_BUILD_PATH} --action_env=TEMP=${TEMP} --action_env=TMP=${TMP}"
|
||||
else
|
||||
BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl --copt=-fvisibility=hidden"
|
||||
fi
|
||||
|
||||
### Define build targets that we will re-ues in sourcing scripts.
|
||||
BUILD_TARGET_LIB_CPP_API="//tensorflow:tensorflow_cc"
|
||||
BUILD_TARGET_GRAPH_TRANSFORMS="//tensorflow/tools/graph_transforms:transform_graph"
|
||||
BUILD_TARGET_GRAPH_SUMMARIZE="//tensorflow/tools/graph_transforms:summarize_graph"
|
||||
BUILD_TARGET_GRAPH_BENCHMARK="//tensorflow/tools/benchmark:benchmark_model"
|
||||
#BUILD_TARGET_CONVERT_MMAP="//tensorflow/contrib/util:convert_graphdef_memmapped_format"
|
||||
BUILD_TARGET_TOCO="//tensorflow/lite/toco:toco"
|
||||
BUILD_TARGET_LITE_BENCHMARK="//tensorflow/lite/tools/benchmark:benchmark_model"
|
||||
BUILD_TARGET_LITE_LIB="//tensorflow/lite/c:libtensorflowlite_c.so"
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-win-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.win_amd64_cpu.url}
|
||||
artifact_namespace: ${system.tensorflow.win_amd64_cpu.namespace}
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_win.pacman} && ${tensorflow.packages_win.msys64}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh"
|
||||
build: "taskcluster/tf_tc-build.sh"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 14400
|
||||
metadata:
|
||||
name: "TensorFlow Windows AMD64 CPU"
|
||||
description: "Building TensorFlow for Windows AMD64, CPU only, optimized version"
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
build:
|
||||
template_file: generic_tc_caching-win-opt-base.tyml
|
||||
cache:
|
||||
artifact_url: ${system.tensorflow.win_amd64_cuda.url}
|
||||
artifact_namespace: ${system.tensorflow.win_amd64_cuda.namespace}
|
||||
system_config:
|
||||
>
|
||||
${tensorflow.packages_win.pacman} && ${tensorflow.packages_win.msys64}
|
||||
scripts:
|
||||
setup: "taskcluster/tf_tc-setup.sh"
|
||||
build: "taskcluster/tf_tc-build.sh --gpu"
|
||||
package: "taskcluster/tf_tc-package.sh"
|
||||
maxRunTime: 14400
|
||||
metadata:
|
||||
name: "TensorFlow Windows AMD64 CUDA"
|
||||
description: "Building TensorFlow for Windows AMD64, CUDA, optimized version"
|
||||
|
|
@ -8,7 +8,8 @@ build:
|
|||
- "node-gyp-cache"
|
||||
- "swig-win-amd64"
|
||||
- "pyenv-win-amd64"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz"
|
||||
- "tf_win-amd64-cpu-opt"
|
||||
tensorflow: ${system.tensorflow.win_amd64_cpu.url}
|
||||
scripts:
|
||||
build: "taskcluster/win-build.sh"
|
||||
package: "taskcluster/win-package.sh"
|
||||
|
|
|
|||
|
|
@ -4,11 +4,12 @@ build:
|
|||
- "swig-win-amd64"
|
||||
- "node-gyp-cache"
|
||||
- "pyenv-win-amd64"
|
||||
- "tf_win-amd64-cpu-opt"
|
||||
routes:
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win-ctc"
|
||||
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win-ctc"
|
||||
- "index.project.deepspeech.deepspeech.native_client.win-ctc.${event.head.sha}"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz"
|
||||
tensorflow: ${system.tensorflow.win_amd64_cpu.url}
|
||||
scripts:
|
||||
build: 'taskcluster/decoder-build.sh'
|
||||
package: 'taskcluster/decoder-package.sh'
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ build:
|
|||
- "node-gyp-cache"
|
||||
- "swig-win-amd64"
|
||||
- "pyenv-win-amd64"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win-cuda/artifacts/public/home.tar.xz"
|
||||
- "tf_win-amd64-gpu-opt"
|
||||
tensorflow: ${system.tensorflow.win_amd64_cuda.url}
|
||||
scripts:
|
||||
build: "taskcluster/win-build.sh --cuda"
|
||||
package: "taskcluster/win-package.sh"
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ build:
|
|||
- "node-gyp-cache"
|
||||
- "swig-win-amd64"
|
||||
- "pyenv-win-amd64"
|
||||
tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz"
|
||||
- "tf_win-amd64-cpu-opt"
|
||||
tensorflow: ${system.tensorflow.win_amd64_cpu.url}
|
||||
scripts:
|
||||
build: "taskcluster/win-build.sh --tflite"
|
||||
package: "taskcluster/win-package.sh"
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ package_option=$1
|
|||
|
||||
source $(dirname "$0")/tc-tests-utils.sh
|
||||
|
||||
source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh
|
||||
source $(dirname "$0")/tf_tc-vars.sh
|
||||
|
||||
BAZEL_TARGETS="
|
||||
//native_client:libdeepspeech.so
|
||||
|
|
@ -31,7 +31,7 @@ SYSTEM_TARGET=host-win
|
|||
do_bazel_build
|
||||
|
||||
if [ "${package_option}" = "--cuda" ]; then
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/libdeepspeech.so.if.lib
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin/native_client/libdeepspeech.so.if.lib
|
||||
fi
|
||||
|
||||
export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin:'/c/Program Files/nodejs/'
|
||||
|
|
|
|||
|
|
@ -40,8 +40,6 @@ payload:
|
|||
"C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz |
|
||||
"C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
|
||||
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
|
||||
- echo .\msys64\usr\bin\bash.exe --login -cxe "
|
||||
export LC_ALL=C &&
|
||||
|
|
@ -54,9 +52,10 @@ payload:
|
|||
(7z x -txz -so $USERPROFILE/home.tar.xz | 7z x -aoa -ttar -si ) &&
|
||||
git clone --quiet $EXAMPLES_CLONE_URL $TASKCLUSTER_TASK_DIR/DeepSpeech/examples &&
|
||||
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && git checkout --quiet $EXAMPLES_CHECKOUT_TARGET &&
|
||||
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
|
||||
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
|
||||
ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
|
||||
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} &&
|
||||
git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ &&
|
||||
(rm $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tensorflow/native_client || true) &&
|
||||
ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tensorflow/native_client &&
|
||||
cd $TASKCLUSTER_TASK_DIR &&
|
||||
(mkdir pyenv-root/ && 7z x -so $USERPROFILE/pyenv.tar.gz | 7z x -opyenv-root/ -aoa -ttar -si ) &&
|
||||
pacman --noconfirm -S tar make &&
|
||||
|
|
@ -75,10 +74,6 @@ payload:
|
|||
content:
|
||||
sha256: ${system.msys2.sha}
|
||||
url: ${system.msys2.url}
|
||||
- file: filesystem-2020.02-3-x86_64.pkg.tar.xz
|
||||
content:
|
||||
sha256: ${system.msys2_filesystem_pkg.sha}
|
||||
url: ${system.msys2_filesystem_pkg.url}
|
||||
- file: home.tar.xz
|
||||
content:
|
||||
url: ${build.tensorflow}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh
|
|||
|
||||
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
|
||||
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/
|
||||
|
||||
package_native_client "native_client.tar.xz"
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 7ead55807a2ded84c107720ebca61e6285e2c239
|
||||
Loading…
Reference in New Issue