From 6d1f0c73ef1bcd0be648937d40910e8dadf6cfb3 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 23 Jun 2020 19:28:16 +0200 Subject: [PATCH 01/20] Bump VERSION to 0.9.0-alpha.0 --- training/deepspeech_training/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/training/deepspeech_training/VERSION b/training/deepspeech_training/VERSION index 0a1ffad4..11e5891f 100644 --- a/training/deepspeech_training/VERSION +++ b/training/deepspeech_training/VERSION @@ -1 +1 @@ -0.7.4 +0.9.0-alpha.0 From 91b3db33c3f7db7d4968e196f8f1b45d38c95e5d Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 26 Jun 2020 15:07:18 +0200 Subject: [PATCH 02/20] Build kenlm in training container image. --- Dockerfile.train.tmpl | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Dockerfile.train.tmpl b/Dockerfile.train.tmpl index 7feab79e..732ebbb0 100644 --- a/Dockerfile.train.tmpl +++ b/Dockerfile.train.tmpl @@ -10,9 +10,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ apt-utils \ bash-completion \ build-essential \ + cmake \ curl \ git \ git-lfs \ + libboost-all-dev \ libbz2-dev \ locales \ python3-venv \ @@ -50,4 +52,16 @@ RUN DS_NODECODER=y DS_NOTENSORFLOW=y pip3 install --upgrade -e . RUN python3 util/taskcluster.py --source tensorflow --branch r1.15 \ --artifact convert_graphdef_memmapped_format --target . +# Build KenLM to generate new scorers +WORKDIR /DeepSpeech/native_client +RUN rm -rf kenlm && \ + git clone https://github.com/kpu/kenlm && \ + cd kenlm && \ + git checkout 87e85e66c99ceff1fab2500a7c60c01da7315eec && \ + mkdir -p build && \ + cd build && \ + cmake .. && \ + make -j $(nproc) +WORKDIR /DeepSpeech + RUN ./bin/run-ldc93s1.sh From eebf12134ed61abfc9a67af1552942a7e4683bec Mon Sep 17 00:00:00 2001 From: Tilman Kamp <5991088+tilmankamp@users.noreply.github.com> Date: Wed, 17 Jun 2020 17:59:36 +0200 Subject: [PATCH 03/20] Warp augmentation --- bin/run-tc-graph_augmentations.sh | 1 + doc/TRAINING.rst | 15 ++++++++++ .../deepspeech_training/util/augmentations.py | 29 +++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/bin/run-tc-graph_augmentations.sh b/bin/run-tc-graph_augmentations.sh index 0d949125..9b6181ae 100755 --- a/bin/run-tc-graph_augmentations.sh +++ b/bin/run-tc-graph_augmentations.sh @@ -20,6 +20,7 @@ python -u DeepSpeech.py --noshow_progressbar --noearly_stop \ --augment dropout \ --augment pitch \ --augment tempo \ + --augment warp \ --augment time_mask \ --augment frequency_mask \ --augment add \ diff --git a/doc/TRAINING.rst b/doc/TRAINING.rst index 479d93dd..17aa8ac1 100644 --- a/doc/TRAINING.rst +++ b/doc/TRAINING.rst @@ -399,6 +399,20 @@ Spectrogram domain augmentations * **factor**: speed factor by which the time axis is stretched or shrunken (e.g. a value of 2.0 will double playback tempo) +**Warp augmentation** ``--augment warp[p=,nt=,nf=,wt=,wf=]`` + Applies a non-linear image warp to the spectrogram. This is achieved by randomly shifting a grid of equally distributed warp points along time and frequency axis. + + * **p**: probability value between 0.0 (never) and 1.0 (always) if a given sample gets augmented by this method + + * **nt**: number of equally distributed warp grid lines along time axis of the spectrogram (excluding the edges) + + * **nf**: number of equally distributed warp grid lines along frequency axis of the spectrogram (excluding the edges) + + * **wt**: standard deviation of the random shift applied to warp points along time axis (0.0 = no warp, 1.0 = half the distance to the neighbour point) + + * **wf**: standard deviation of the random shift applied to warp points along frequency axis (0.0 = no warp, 1.0 = half the distance to the neighbour point) + + **Frequency mask augmentation** ``--augment frequency_mask[p=,n=,size=]`` Sets frequency-intervals within the augmented samples to zero (silence) at random frequencies. See the SpecAugment paper for more details - https://arxiv.org/abs/1904.08779 @@ -469,6 +483,7 @@ Example training with all augmentations: --augment volume[p=0.1,dbfs=-10:-40] \ --augment pitch[p=0.1,pitch=1~0.2] \ --augment tempo[p=0.1,factor=1~0.5] \ + --augment warp[p=0.1,nt=4,nf=1,wt=0.5:1.0,wf=0.1:0.2] \ --augment frequency_mask[p=0.1,n=1:3,size=1:5] \ --augment time_mask[p=0.1,domain=signal,n=3:10~2,size=50:100~40] \ --augment dropout[p=0.1,rate=0.05] \ diff --git a/training/deepspeech_training/util/augmentations.py b/training/deepspeech_training/util/augmentations.py index b7033c51..eef58b81 100644 --- a/training/deepspeech_training/util/augmentations.py +++ b/training/deepspeech_training/util/augmentations.py @@ -412,6 +412,35 @@ class Tempo(GraphAugmentation): return spectrogram_aug[:, :, :, 0] +class Warp(GraphAugmentation): + """See "Warp augmentation" in training documentation""" + def __init__(self, p=1.0, nt=1, nf=1, wt=0.1, wf=0.0): + super(Warp, self).__init__(p, domain='spectrogram') + self.num_t = int_range(nt) + self.num_f = int_range(nf) + self.warp_t = float_range(wt) + self.warp_f = float_range(wf) + + def apply(self, tensor, transcript=None, clock=0.0): + import tensorflow as tf # pylint: disable=import-outside-toplevel + original_shape = tf.shape(tensor) + size_t, size_f = original_shape[1], original_shape[2] + seed = (clock * tf.int32.min, clock * tf.int32.max) + num_t = tf_pick_value_from_range(self.num_t, clock=clock) + num_f = tf_pick_value_from_range(self.num_f, clock=clock) + + def get_flows(n, size, warp): + warp = tf_pick_value_from_range(warp, clock=clock) + warp = warp * tf.cast(size, dtype=tf.float32) / tf.cast(2 * (n + 1), dtype=tf.float32) + f = tf.random.stateless_normal([num_t, num_f], seed, mean=0.0, stddev=warp, dtype=tf.float32) + return tf.pad(f, tf.constant([[1, 1], [1, 1]]), 'CONSTANT') # zero flow at all edges + + flows = tf.stack([get_flows(num_t, size_t, self.warp_t), get_flows(num_f, size_f, self.warp_f)], axis=2) + flows = tf.image.resize_bicubic(tf.expand_dims(flows, 0), [size_t, size_f]) + spectrogram_aug = tf.contrib.image.dense_image_warp(tf.expand_dims(tensor, -1), flows) + return tf.reshape(spectrogram_aug, shape=(1, -1, size_f)) + + class FrequencyMask(GraphAugmentation): """See "Frequency mask augmentation" in training documentation""" def __init__(self, p=1.0, n=3, size=2): From 80ee63fac68354bf872693c3ad672427266ae737 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Thu, 25 Jun 2020 14:44:31 +0200 Subject: [PATCH 04/20] Use TensorFlow as a submodule --- .gitmodules | 3 + Dockerfile.build.tmpl | 18 +- native_client/README.rst | 30 ++- native_client/definitions.mk | 2 +- native_client/dotnet/README.rst | 22 +- taskcluster/.build.yml | 5 +- taskcluster/.shared.yml | 42 +++- taskcluster/android-apk-build.sh | 2 +- taskcluster/android-apk-package.sh | 2 +- taskcluster/android-arm64-cpu-opt.yml | 4 +- taskcluster/android-armv7-cpu-opt.yml | 3 +- taskcluster/android-build.sh | 2 +- .../android-cache-arm64-v8a-android-24.yml | 1 + .../android-cache-arm64-v8a-android-25.yml | 1 + .../android-cache-armeabi-v7a-android-24.yml | 1 + .../android-cache-armeabi-v7a-android-25.yml | 1 + taskcluster/android-cache-sdk-android-27.yml | 1 + .../android-cache-x86_64-android-24.yml | 1 + .../android-cache-x86_64-android-25.yml | 1 + .../android-cache-x86_64-android-26.yml | 1 + .../android-cache-x86_64-android-28.yml | 1 + .../android-cache-x86_64-android-29.yml | 1 + .../android-cache-x86_64-android-30.yml | 1 + taskcluster/android-java-opt.yml | 2 +- taskcluster/android-package.sh | 2 +- taskcluster/android-x86_64-cpu-opt.yml | 2 +- taskcluster/arm64-build.sh | 2 +- taskcluster/cuda-build.sh | 2 +- taskcluster/darwin-amd64-cpu-opt.yml | 3 +- taskcluster/darwin-amd64-ctc-opt.yml | 3 +- taskcluster/darwin-amd64-tflite-opt.yml | 3 +- taskcluster/darwin-opt-base.tyml | 5 +- taskcluster/decoder-build.sh | 2 +- .../generic_tc_caching-darwin-opt-base.tyml | 7 +- .../generic_tc_caching-linux-opt-base.tyml | 4 +- .../generic_tc_caching-win-opt-base.tyml | 11 +- taskcluster/gradle-cache.yml | 1 + taskcluster/homebrew_builds-darwin-amd64.yml | 1 + taskcluster/homebrew_tests-darwin-amd64.yml | 1 + taskcluster/host-build.sh | 2 +- taskcluster/linux-amd64-cpu-opt.yml | 4 +- taskcluster/linux-amd64-ctc-opt.yml | 4 +- taskcluster/linux-amd64-gpu-opt.yml | 4 +- taskcluster/linux-amd64-tflite-opt.yml | 4 +- taskcluster/linux-arm64-cpu-opt.yml | 4 +- taskcluster/linux-opt-base.tyml | 4 +- taskcluster/linux-rpi3-cpu-opt.yml | 4 +- taskcluster/node-build.sh | 2 +- taskcluster/node-gyp-cache.yml | 1 + taskcluster/node-package-opt-base.tyml | 2 +- taskcluster/package.sh | 2 +- taskcluster/pyenv-darwin-amd64.yml | 1 + taskcluster/pyenv-linux-amd64.yml | 1 + taskcluster/pyenv-win-amd64.yml | 1 + taskcluster/rpi3-build.sh | 2 +- taskcluster/swig-darwin-amd64.yml | 1 + taskcluster/swig-linux-amd64.yml | 1 + taskcluster/swig-win-amd64.yml | 1 + taskcluster/tc-all-utils.sh | 8 +- taskcluster/tc-all-vars.sh | 2 +- taskcluster/tc-build-utils.sh | 10 +- taskcluster/tc-decision.py | 238 ++++++++++++++++++ taskcluster/tc-decision_reqs.txt | 5 + taskcluster/tc-schedule.sh | 6 +- taskcluster/test-win-cuda-opt-base.tyml | 6 - taskcluster/test-win-opt-base.tyml | 6 - taskcluster/tf_android-arm64-opt.yml | 16 ++ taskcluster/tf_android-armv7-opt.yml | 16 ++ taskcluster/tf_darwin-amd64-opt.yml | 18 ++ taskcluster/tf_linux-amd64-cpu-opt.yml | 16 ++ taskcluster/tf_linux-amd64-gpu-opt.yml | 16 ++ taskcluster/tf_linux-arm64-cpu-opt.yml | 16 ++ taskcluster/tf_linux-rpi3-cpu-opt.yml | 16 ++ taskcluster/tf_tc-brew.sh | 44 ++++ taskcluster/tf_tc-build.sh | 108 ++++++++ taskcluster/tf_tc-package.sh | 62 +++++ taskcluster/tf_tc-pip.sh | 7 + taskcluster/tf_tc-setup.sh | 114 +++++++++ taskcluster/tf_tc-vars.sh | 195 ++++++++++++++ taskcluster/tf_win-amd64-cpu-opt.yml | 16 ++ taskcluster/tf_win-amd64-gpu-opt.yml | 16 ++ taskcluster/win-amd64-cpu-opt.yml | 3 +- taskcluster/win-amd64-ctc-opt.yml | 3 +- taskcluster/win-amd64-gpu-opt.yml | 3 +- taskcluster/win-amd64-tflite-opt.yml | 3 +- taskcluster/win-build.sh | 4 +- taskcluster/win-opt-base.tyml | 13 +- taskcluster/win-package.sh | 2 +- tensorflow | 1 + 89 files changed, 1102 insertions(+), 129 deletions(-) create mode 100644 taskcluster/tc-decision.py create mode 100644 taskcluster/tc-decision_reqs.txt create mode 100644 taskcluster/tf_android-arm64-opt.yml create mode 100644 taskcluster/tf_android-armv7-opt.yml create mode 100644 taskcluster/tf_darwin-amd64-opt.yml create mode 100644 taskcluster/tf_linux-amd64-cpu-opt.yml create mode 100644 taskcluster/tf_linux-amd64-gpu-opt.yml create mode 100644 taskcluster/tf_linux-arm64-cpu-opt.yml create mode 100644 taskcluster/tf_linux-rpi3-cpu-opt.yml create mode 100755 taskcluster/tf_tc-brew.sh create mode 100755 taskcluster/tf_tc-build.sh create mode 100755 taskcluster/tf_tc-package.sh create mode 100755 taskcluster/tf_tc-pip.sh create mode 100755 taskcluster/tf_tc-setup.sh create mode 100755 taskcluster/tf_tc-vars.sh create mode 100644 taskcluster/tf_win-amd64-cpu-opt.yml create mode 100644 taskcluster/tf_win-amd64-gpu-opt.yml create mode 160000 tensorflow diff --git a/.gitmodules b/.gitmodules index fc8a954f..70cd92a1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = doc/examples url = https://github.com/mozilla/DeepSpeech-examples.git branch = master +[submodule "tensorflow"] + path = tensorflow + url = https://github.com/mozilla/tensorflow.git diff --git a/Dockerfile.build.tmpl b/Dockerfile.build.tmpl index 73c8bdd2..ccea956c 100644 --- a/Dockerfile.build.tmpl +++ b/Dockerfile.build.tmpl @@ -53,11 +53,6 @@ RUN dpkg -i bazel_*.deb # >> START Configure Tensorflow Build -# Clone TensorFlow from Mozilla repo -RUN git clone https://github.com/mozilla/tensorflow/ -WORKDIR /tensorflow -RUN git checkout r2.2 - # GPU Environment Setup ENV TF_NEED_ROCM 0 ENV TF_NEED_OPENCL_SYCL 0 @@ -116,16 +111,15 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ WORKDIR / -RUN git clone $DEEPSPEECH_REPO +RUN git clone --recursive $DEEPSPEECH_REPO WORKDIR /DeepSpeech RUN git checkout $DEEPSPEECH_SHA - -# Link DeepSpeech native_client libs to tf folder -RUN ln -s /DeepSpeech/native_client /tensorflow +RUN git submodule sync tensorflow/ +RUN git submodule update --init tensorflow/ # >> START Build and bind -WORKDIR /tensorflow +WORKDIR /DeepSpeech/tensorflow # Fix for not found script https://github.com/tensorflow/tensorflow/issues/471 RUN ./configure @@ -158,10 +152,10 @@ RUN bazel build \ --action_env=LD_LIBRARY_PATH=${LD_LIBRARY_PATH} # Copy built libs to /DeepSpeech/native_client -RUN cp /tensorflow/bazel-bin/native_client/libdeepspeech.so /DeepSpeech/native_client/ +RUN cp bazel-bin/native_client/libdeepspeech.so /DeepSpeech/native_client/ # Build client.cc and install Python client and decoder bindings -ENV TFDIR /tensorflow +ENV TFDIR /DeepSpeech/tensorflow RUN nproc diff --git a/native_client/README.rst b/native_client/README.rst index d77df35d..fb1c2d0e 100644 --- a/native_client/README.rst +++ b/native_client/README.rst @@ -4,10 +4,8 @@ Building DeepSpeech Binaries If you'd like to build the DeepSpeech binaries yourself, you'll need the following pre-requisites downloaded and installed: - -* `Mozilla's TensorFlow r2.2 branch `_ * `Bazel 2.0.0 `_ -* `General TensorFlow requirements `_ +* `General TensorFlow r2.2 requirements `_ * `libsox `_ It is required to use our fork of TensorFlow since it includes fixes for common problems encountered when building the native client files. @@ -28,15 +26,16 @@ If you follow these instructions, you should compile your own binaries of DeepSp For more information on configuring TensorFlow, read the docs up to the end of `"Configure the Build" `_. -TensorFlow: Clone & Checkout -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Checkout source code +^^^^^^^^^^^^^^^^^^^^ -Clone our fork of TensorFlow and checkout the correct version: +Clone DeepSpeech source code (TensorFlow will come as a submdule): .. code-block:: - git clone https://github.com/mozilla/tensorflow.git - git checkout origin/r2.2 + git clone https://github.com/mozilla/DeepSpeech.git + git submodule sync tensorflow/ + git submodule update --init tensorflow/ Bazel: Download & Install ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -57,16 +56,16 @@ Compile DeepSpeech ------------------ Compile ``libdeepspeech.so`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Within your TensorFlow checkout, create a symbolic link to the DeepSpeech ``native_client`` directory. Assuming DeepSpeech and TensorFlow checkouts are in the same directory, do: +Within your TensorFlow directory, there should be a symbolic link to the DeepSpeech ``native_client`` directory. If it is not present, create it with the follow command: .. code-block:: cd tensorflow - ln -s ../DeepSpeech/native_client ./ + ln -s ../native_client -You can now use Bazel to build the main DeepSpeech library, ``libdeepspeech.so``\ . Add ``--config=cuda`` if you want a CUDA build. +You can now use Bazel to build the main DeepSpeech library, ``libdeepspeech.so``. Add ``--config=cuda`` if you want a CUDA build. .. code-block:: @@ -77,11 +76,10 @@ The generated binaries will be saved to ``bazel-bin/native_client/``. Compile Language Bindings ^^^^^^^^^^^^^^^^^^^^^^^^^ -Now, ``cd`` into the ``DeepSpeech/native_client`` directory and use the ``Makefile`` to build all the language bindings (C++ client, Python package, Nodejs package, etc.). Set the environment variable ``TFDIR`` to point to your TensorFlow checkout. +Now, ``cd`` into the ``DeepSpeech/native_client`` directory and use the ``Makefile`` to build all the language bindings (C++ client, Python package, Nodejs package, etc.). .. code-block:: - TFDIR=~/tensorflow cd ../DeepSpeech/native_client make deepspeech @@ -191,11 +189,11 @@ Building the ``deepspeech`` binary will happen through ``ndk-build`` (ARMv7): .. code-block:: cd ../DeepSpeech/native_client - $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../../tensorflow/ TARGET_ARCH_ABI=armeabi-v7a + $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../tensorflow/ TARGET_ARCH_ABI=armeabi-v7a And (ARM64): .. code-block:: cd ../DeepSpeech/native_client - $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../../tensorflowx/ TARGET_ARCH_ABI=arm64-v8a + $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../tensorflow/ TARGET_ARCH_ABI=arm64-v8a diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 8d43ce0a..2f6afbf4 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -1,7 +1,7 @@ NC_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) TARGET ?= host -TFDIR ?= $(abspath $(NC_DIR)/../../tensorflow) +TFDIR ?= $(abspath $(NC_DIR)/../tensorflow) PREFIX ?= /usr/local SO_SEARCH ?= $(TFDIR)/bazel-bin/ diff --git a/native_client/dotnet/README.rst b/native_client/dotnet/README.rst index 97ac0475..9f50f446 100644 --- a/native_client/dotnet/README.rst +++ b/native_client/dotnet/README.rst @@ -43,36 +43,34 @@ We highly recommend sticking to the recommended versions of CUDA/cuDNN in order Getting the code ---------------- -We need to clone ``mozilla/DeepSpeech`` and ``mozilla/tensorflow``. +We need to clone ``mozilla/DeepSpeech``. .. code-block:: bash git clone https://github.com/mozilla/DeepSpeech - -.. code-block:: bash - - git clone --branch r2.2 https://github.com/mozilla/tensorflow + git submodule sync tensorflow/ + git submodule update --init tensorflow/ Configuring the paths --------------------- -We need to create a symbolic link, for this example let's suppose that we cloned into ``D:\cloned`` and now the structure looks like: +There should already be a symbolic link, for this example let's suppose that we cloned into ``D:\cloned`` and now the structure looks like: .. code-block:: . ├── D:\ │ ├── cloned # Contains DeepSpeech and tensorflow side by side - │ │ ├── DeepSpeech # Root of the cloned DeepSpeech - │ │ ├── tensorflow # Root of the cloned Mozilla's tensorflow + │ │ └── DeepSpeech # Root of the cloned DeepSpeech + │ │ ├── tensorflow # Root of the cloned Mozilla's tensorflow └── ... -Change your path accordingly to your path structure, for the structure above we are going to use the following command: +Change your path accordingly to your path structure, for the structure above we are going to use the following command if the symbolic link does not exists: .. code-block:: bash - mklink /d "D:\cloned\tensorflow\native_client" "D:\cloned\DeepSpeech\native_client" + mklink /d "D:\cloned\DeepSpeech\tensorflow\native_client" "D:\cloned\DeepSpeech\native_client" Adding environment variables ---------------------------- @@ -82,7 +80,7 @@ After you have installed the requirements there are few environment variables th MSYS2 paths ~~~~~~~~~~~ -For MSYS2 we need to add ``bin`` directory, if you installed in the default route the path that we need to add should looks like ``C:\msys64\usr\bin``. Now we can run ``pacman``\ : +For MSYS2 we need to add ``bin`` directory, if you installed in the default route the path that we need to add should looks like ``C:\msys64\usr\bin``. Now we can run ``pacman``: .. code-block:: bash @@ -120,7 +118,7 @@ Building the native_client There's one last command to run before building, you need to run the `configure.py `_ inside ``tensorflow`` cloned directory. -At this point we are ready to start building the ``native_client``\ , go to ``tensorflow`` directory that you cloned, following our examples should be ``D:\cloned\tensorflow``. +At this point we are ready to start building the ``native_client``, go to ``tensorflow`` sub-directory, following our examples should be ``D:\cloned\DeepSpeech\tensorflow``. CPU ~~~ diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 1b94c52d..7fda06e6 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -10,6 +10,8 @@ build: routes: [] maxRunTime: 3600 docker_image: "ubuntu:16.04" + generic: + workerType: 'ds-macos-light' system_setup: > true @@ -17,12 +19,13 @@ build: > true scripts: + setup: '' build: '' package: '' nc_asset_name: 'native_client.tar.xz' args: tests_cmdline: '' - tensorflow_git_desc: 'TensorFlow: v2.2.0-12-gc29895f' + tensorflow_git_desc: 'TensorFlow: v2.2.0-14-g7ead558' test_model_task: '' homebrew: url: '' diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index f6c10848..dcae80b7 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -8,9 +8,17 @@ python: training: packages_xenial: apt: 'libopus0' +deepspeech: + packages_xenial: + apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python python-dev zlib1g-dev' tensorflow: packages_xenial: - apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python zlib1g-dev' + apt: 'apt-get -qq update && apt-get -qq -y install realpath build-essential python-virtualenv python-dev python-pip libblas-dev liblapack-dev gfortran wget software-properties-common pixz zip zlib1g-dev unzip' + packages_macos: + brew: '$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tf_tc-brew.sh' + packages_win: + pacman: 'pacman --noconfirm -S patch unzip tar' + msys64: 'ln -s $USERPROFILE/msys64 $TASKCLUSTER_TASK_DIR/msys64' java: packages_xenial: apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f' @@ -132,6 +140,34 @@ system: win: url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118/artifacts/public/ds-swig.tar.gz" namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" + tensorflow: + linux_amd64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cpu/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cpu" + linux_amd64_cuda: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cuda" + linux_armv7: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm" + linux_arm64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm64" + darwin_amd64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.osx/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.osx" + android_arm64: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-arm64" + android_armv7: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-armv7" + win_amd64_cpu: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win" + win_amd64_cuda: + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win-cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win-cuda" username: 'build-user' homedir: linux: '/home/build-user' @@ -141,7 +177,3 @@ system: msys2: url: 'https://github.com/msys2/msys2-installer/releases/download/2020-06-02/msys2-base-x86_64-20200602.tar.xz' sha: '598ceeaa3e2ccf86a25a2e3c449d00a9fd35300e36011bee610036dfa59d670a' - msys2_filesystem_pkg: - url: 'http://repo.msys2.org/msys/x86_64/filesystem-2020.02-3-x86_64.pkg.tar.xz' - sha: '927b020a67a05139ee1b2c45bff491c1d42335e64350cc7758ee20d7c3099477' - install: 'pacman -Udd --noconfirm $USERPROFILE/filesystem-2020.02-3-x86_64.pkg.tar.xz' diff --git a/taskcluster/android-apk-build.sh b/taskcluster/android-apk-build.sh index 9259995e..1590b526 100755 --- a/taskcluster/android-apk-build.sh +++ b/taskcluster/android-apk-build.sh @@ -6,6 +6,6 @@ arm_flavor=$1 source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh do_deepspeech_java_apk_build diff --git a/taskcluster/android-apk-package.sh b/taskcluster/android-apk-package.sh index 64262c7c..40ab343f 100755 --- a/taskcluster/android-apk-package.sh +++ b/taskcluster/android-apk-package.sh @@ -8,7 +8,7 @@ source $(dirname "$0")/tc-tests-utils.sh mkdir -p ${TASKCLUSTER_ARTIFACTS} || true -cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ +cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/java/app/build/outputs/apk/release/app*.apk ${TASKCLUSTER_ARTIFACTS}/ cp ${DS_ROOT_TASK}/DeepSpeech/ds/native_client/java/libdeepspeech/build/outputs/aar/libdeepspeech*.aar ${TASKCLUSTER_ARTIFACTS}/ diff --git a/taskcluster/android-arm64-cpu-opt.yml b/taskcluster/android-arm64-cpu-opt.yml index 43b756cc..abc55412 100644 --- a/taskcluster/android-arm64-cpu-opt.yml +++ b/taskcluster/android-arm64-cpu-opt.yml @@ -4,12 +4,14 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_android-arm64-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64" - "index.project.deepspeech.deepspeech.native_client.android-arm64.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-arm64/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.android_arm64.url} scripts: + setup: "" build: "taskcluster/android-build.sh arm64-v8a" package: "taskcluster/android-package.sh arm64-v8a" nc_asset_name: "native_client.arm64.cpu.android.tar.xz" diff --git a/taskcluster/android-armv7-cpu-opt.yml b/taskcluster/android-armv7-cpu-opt.yml index 168b542b..196f3e6d 100644 --- a/taskcluster/android-armv7-cpu-opt.yml +++ b/taskcluster/android-armv7-cpu-opt.yml @@ -4,11 +4,12 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_android-armv7-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7" - "index.project.deepspeech.deepspeech.native_client.android-armv7.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-armv7/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.android_armv7.url} scripts: build: "taskcluster/android-build.sh armeabi-v7a" package: "taskcluster/android-package.sh armeabi-v7a" diff --git a/taskcluster/android-build.sh b/taskcluster/android-build.sh index ab089459..0121e51d 100644 --- a/taskcluster/android-build.sh +++ b/taskcluster/android-build.sh @@ -6,7 +6,7 @@ arm_flavor=$1 source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so diff --git a/taskcluster/android-cache-arm64-v8a-android-24.yml b/taskcluster/android-cache-arm64-v8a-android-24.yml index 260ea546..35659584 100644 --- a/taskcluster/android-cache-arm64-v8a-android-24.yml +++ b/taskcluster/android-cache-arm64-v8a-android-24.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.arm64_v8a.android_24.url} artifact_namespace: ${system.android_cache.arm64_v8a.android_24.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh arm64-v8a android-24" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-arm64-v8a-android-25.yml b/taskcluster/android-cache-arm64-v8a-android-25.yml index 34b8158b..e8f81091 100644 --- a/taskcluster/android-cache-arm64-v8a-android-25.yml +++ b/taskcluster/android-cache-arm64-v8a-android-25.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.arm64_v8a.android_25.url} artifact_namespace: ${system.android_cache.arm64_v8a.android_25.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh arm64-v8a android-25" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-armeabi-v7a-android-24.yml b/taskcluster/android-cache-armeabi-v7a-android-24.yml index 441b9255..b2232534 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-24.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-24.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.armeabi_v7a.android_24.url} artifact_namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh armeabi-v7a android-24 default" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-armeabi-v7a-android-25.yml b/taskcluster/android-cache-armeabi-v7a-android-25.yml index 620745a0..d35c9ab5 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-25.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-25.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.armeabi_v7a.android_25.url} artifact_namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh armeabi-v7a android-25" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-sdk-android-27.yml b/taskcluster/android-cache-sdk-android-27.yml index 0b047dc1..244592e2 100644 --- a/taskcluster/android-cache-sdk-android-27.yml +++ b/taskcluster/android-cache-sdk-android-27.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.sdk.android_27.url} artifact_namespace: ${system.android_cache.sdk.android_27.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh sdk android-27" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-x86_64-android-24.yml b/taskcluster/android-cache-x86_64-android-24.yml index c3b3ba1d..95712b8a 100644 --- a/taskcluster/android-cache-x86_64-android-24.yml +++ b/taskcluster/android-cache-x86_64-android-24.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.x86_64.android_24.url} artifact_namespace: ${system.android_cache.x86_64.android_24.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh x86_64 android-24" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-x86_64-android-25.yml b/taskcluster/android-cache-x86_64-android-25.yml index a2e642e6..3b49de87 100644 --- a/taskcluster/android-cache-x86_64-android-25.yml +++ b/taskcluster/android-cache-x86_64-android-25.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.x86_64.android_25.url} artifact_namespace: ${system.android_cache.x86_64.android_25.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh x86_64 android-25" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-x86_64-android-26.yml b/taskcluster/android-cache-x86_64-android-26.yml index a8b9b15a..d63298b4 100644 --- a/taskcluster/android-cache-x86_64-android-26.yml +++ b/taskcluster/android-cache-x86_64-android-26.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.x86_64.android_26.url} artifact_namespace: ${system.android_cache.x86_64.android_26.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh x86_64 android-26" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-x86_64-android-28.yml b/taskcluster/android-cache-x86_64-android-28.yml index 6193dd00..863620e2 100644 --- a/taskcluster/android-cache-x86_64-android-28.yml +++ b/taskcluster/android-cache-x86_64-android-28.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.x86_64.android_28.url} artifact_namespace: ${system.android_cache.x86_64.android_28.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh x86_64 android-28" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-x86_64-android-29.yml b/taskcluster/android-cache-x86_64-android-29.yml index 63e1b3c6..087cb7b8 100644 --- a/taskcluster/android-cache-x86_64-android-29.yml +++ b/taskcluster/android-cache-x86_64-android-29.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.x86_64.android_29.url} artifact_namespace: ${system.android_cache.x86_64.android_29.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh x86_64 android-29" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-cache-x86_64-android-30.yml b/taskcluster/android-cache-x86_64-android-30.yml index 51cf50b9..a4b8221c 100644 --- a/taskcluster/android-cache-x86_64-android-30.yml +++ b/taskcluster/android-cache-x86_64-android-30.yml @@ -7,6 +7,7 @@ build: artifact_url: ${system.android_cache.x86_64.android_30.url} artifact_namespace: ${system.android_cache.x86_64.android_30.namespace} scripts: + setup: "" build: "taskcluster/android_cache-build.sh x86_64 android-30" package: "taskcluster/android_cache-package.sh" metadata: diff --git a/taskcluster/android-java-opt.yml b/taskcluster/android-java-opt.yml index 5d4ec601..c92bfbc5 100644 --- a/taskcluster/android-java-opt.yml +++ b/taskcluster/android-java-opt.yml @@ -14,7 +14,7 @@ build: system_setup: > ${java.packages_xenial.apt} - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-armv7/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.android_armv7.url} gradle_cache: url: ${system.gradle_cache.url} namespace: ${system.gradle_cache.namespace} diff --git a/taskcluster/android-package.sh b/taskcluster/android-package.sh index 838758b4..5d42e198 100755 --- a/taskcluster/android-package.sh +++ b/taskcluster/android-package.sh @@ -8,6 +8,6 @@ source $(dirname "$0")/tc-tests-utils.sh mkdir -p ${TASKCLUSTER_ARTIFACTS} || true -cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ +cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ package_native_client_ndk "native_client.tar.xz" "${arm_flavor}" diff --git a/taskcluster/android-x86_64-cpu-opt.yml b/taskcluster/android-x86_64-cpu-opt.yml index a304fe9f..092b7955 100644 --- a/taskcluster/android-x86_64-cpu-opt.yml +++ b/taskcluster/android-x86_64-cpu-opt.yml @@ -8,7 +8,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64" - "index.project.deepspeech.deepspeech.native_client.android-x86_64.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-arm64/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.android_arm64.url} scripts: build: "taskcluster/android-build.sh x86_64" package: "taskcluster/android-package.sh x86_64" diff --git a/taskcluster/arm64-build.sh b/taskcluster/arm64-build.sh index dd0f77ff..7cfb7abf 100644 --- a/taskcluster/arm64-build.sh +++ b/taskcluster/arm64-build.sh @@ -4,7 +4,7 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so diff --git a/taskcluster/cuda-build.sh b/taskcluster/cuda-build.sh index df3e049f..dfaa236a 100755 --- a/taskcluster/cuda-build.sh +++ b/taskcluster/cuda-build.sh @@ -4,7 +4,7 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index 12f41b2f..99bb3f36 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -5,11 +5,12 @@ build: - "node-gyp-cache" - "homebrew_builds-darwin-amd64" - "pyenv-darwin-amd64" + - "tf_darwin-amd64-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx" - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.darwin_amd64.url} scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/darwin-amd64-ctc-opt.yml b/taskcluster/darwin-amd64-ctc-opt.yml index 0f80f31e..21e782ba 100644 --- a/taskcluster/darwin-amd64-ctc-opt.yml +++ b/taskcluster/darwin-amd64-ctc-opt.yml @@ -5,11 +5,12 @@ build: - "node-gyp-cache" - "homebrew_builds-darwin-amd64" - "pyenv-darwin-amd64" + - "tf_darwin-amd64-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc" - "index.project.deepspeech.deepspeech.native_client.osx-ctc.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.darwin_amd64.url} maxRunTime: 14400 scripts: build: 'taskcluster/decoder-build.sh' diff --git a/taskcluster/darwin-amd64-tflite-opt.yml b/taskcluster/darwin-amd64-tflite-opt.yml index 4a22e0dc..05af22fa 100644 --- a/taskcluster/darwin-amd64-tflite-opt.yml +++ b/taskcluster/darwin-amd64-tflite-opt.yml @@ -5,11 +5,12 @@ build: - "node-gyp-cache" - "homebrew_builds-darwin-amd64" - "pyenv-darwin-amd64" + - "tf_darwin-amd64-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite" - "index.project.deepspeech.deepspeech.native_client.osx-tflite.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.darwin_amd64.url} scripts: build: "taskcluster/host-build.sh tflite" package: "taskcluster/package.sh" diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index 4500d5fa..1c5b2faf 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -60,9 +60,8 @@ payload: cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ && exit $TASKCLUSTER_TASK_EXIT_CODE" 0 && (pixz -d < $TASKCLUSTER_ORIG_TASKDIR/home.tar.xz | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) && - git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && - cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && - ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client && + cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} && + git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && cd $TASKCLUSTER_TASK_DIR && (mkdir pyenv-root/ && gtar -C pyenv-root/ -xf $TASKCLUSTER_ORIG_TASKDIR/pyenv.tar.gz) && (mkdir homebrew-builds/ && gtar -C homebrew-builds/ -xf $TASKCLUSTER_ORIG_TASKDIR/homebrew-builds.tar.gz) && diff --git a/taskcluster/decoder-build.sh b/taskcluster/decoder-build.sh index 240a57ea..51a80140 100755 --- a/taskcluster/decoder-build.sh +++ b/taskcluster/decoder-build.sh @@ -4,7 +4,7 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then export SYSTEM_TARGET=host-win diff --git a/taskcluster/generic_tc_caching-darwin-opt-base.tyml b/taskcluster/generic_tc_caching-darwin-opt-base.tyml index e6777f22..75bd4193 100644 --- a/taskcluster/generic_tc_caching-darwin-opt-base.tyml +++ b/taskcluster/generic_tc_caching-darwin-opt-base.tyml @@ -1,6 +1,6 @@ taskId: ${taskcluster.taskId} provisionerId: ${taskcluster.generic.provisionerId} -workerType: ${taskcluster.generic.workerType} +workerType: ${build.generic.workerType} taskGroupId: ${taskcluster.taskGroupId} schedulerId: ${taskcluster.schedulerId} dependencies: @@ -24,6 +24,8 @@ payload: - "--login" - "-cxe" - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + extraSystemConfig: { $eval: strip(str(build.system_config)) } taskIndexExpire: { $fromNow: '6 months' } in: > export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && @@ -32,13 +34,14 @@ payload: export TASKCLUSTER_TASK_DIR="$(pwd)" && export LC_ALL=C && export MACOSX_DEPLOYMENT_TARGET=10.10 && - export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && env && mkdir -p $TASKCLUSTER_ARTIFACTS/ && cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` && if [ "$cache_artifact" != "200" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && + ${extraSystemConfig} && + $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.setup} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} 127.0.0.1:8080 ${build.cache.artifact_namespace} diff --git a/taskcluster/generic_tc_caching-linux-opt-base.tyml b/taskcluster/generic_tc_caching-linux-opt-base.tyml index da91f115..7de29ada 100644 --- a/taskcluster/generic_tc_caching-linux-opt-base.tyml +++ b/taskcluster/generic_tc_caching-linux-opt-base.tyml @@ -22,6 +22,7 @@ payload: - "-cxe" - $let: extraSystemSetup: { $eval: strip(str(build.system_setup)) } + extraSystemConfig: { $eval: strip(str(build.system_config)) } taskIndexExpire: { $fromNow: '6 months' } in: > (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && @@ -31,7 +32,8 @@ payload: adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && mkdir -p /tmp/artifacts/ && chmod 777 /tmp/artifacts && echo -e "#!/bin/bash\nset -xe\n env && id && (git clone --quiet ${build.build_or_cache.repo} ~/${build.build_or_cache.dir}/ && cd ~/${build.build_or_cache.dir}/ && git checkout --quiet ${build.build_or_cache.sha})" > /tmp/clone.sh && chmod +x /tmp/clone.sh && - sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && + sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && + sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.setup} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.build} && sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/${build.scripts.package} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace} diff --git a/taskcluster/generic_tc_caching-win-opt-base.tyml b/taskcluster/generic_tc_caching-win-opt-base.tyml index f96d4c6c..43f6372e 100644 --- a/taskcluster/generic_tc_caching-win-opt-base.tyml +++ b/taskcluster/generic_tc_caching-win-opt-base.tyml @@ -20,10 +20,6 @@ payload: content: sha256: ${system.msys2.sha} url: ${system.msys2.url} - - file: filesystem-2020.02-3-x86_64.pkg.tar.xz - content: - sha256: ${system.msys2_filesystem_pkg.sha} - url: ${system.msys2_filesystem_pkg.url} env: TC_MSYS_VERSION: 'MSYS_NT-6.3-9600' @@ -34,18 +30,19 @@ payload: "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0" - - .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}" - - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - $let: + extraSystemSetup: { $eval: strip(str(build.system_setup)) } + extraSystemConfig: { $eval: strip(str(build.system_config)) } taskIndexExpire: { $fromNow: '6 months' } in: > echo .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C && export PATH=\"$USERPROFILE/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && export TASKCLUSTER_ARTIFACTS=\"$(cygpath -u $USERPROFILE/public)\" && export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && + (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && echo \"export TASKCLUSTER_TASK_EXIT_CODE=0\" > $USERPROFILE/tc-exit.sh && - env && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k + env && pacman --noconfirm -S tar && mkdir -p $TASKCLUSTER_ARTIFACTS/ && if [ \"`curl -sSIL -o /dev/null -w %%{http_code} ${build.cache.artifact_url}`\" != \"200\" ]; then git clone --quiet ${build.build_or_cache.repo} $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/ && cd $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir} && git checkout --quiet ${build.build_or_cache.sha} && ${extraSystemConfig} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.setup} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/${build.scripts.package} && $TASKCLUSTER_TASK_DIR/${build.build_or_cache.dir}/taskcluster/tc-update-index.sh ${taskIndexExpire} taskcluster ${build.cache.artifact_namespace}; fi; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh" | cmd /k - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE" diff --git a/taskcluster/gradle-cache.yml b/taskcluster/gradle-cache.yml index a77b2e19..af1f66fa 100644 --- a/taskcluster/gradle-cache.yml +++ b/taskcluster/gradle-cache.yml @@ -7,6 +7,7 @@ build: > ${java.packages_xenial.apt} scripts: + setup: "" build: "taskcluster/gradle-build.sh" package: "taskcluster/gradle-package.sh" metadata: diff --git a/taskcluster/homebrew_builds-darwin-amd64.yml b/taskcluster/homebrew_builds-darwin-amd64.yml index 3377dcac..8846a247 100644 --- a/taskcluster/homebrew_builds-darwin-amd64.yml +++ b/taskcluster/homebrew_builds-darwin-amd64.yml @@ -4,6 +4,7 @@ build: artifact_url: ${system.homebrew_builds.url} artifact_namespace: ${system.homebrew_builds.namespace} scripts: + setup: "" build: "taskcluster/homebrew-build.sh --builds" package: "taskcluster/homebrew-package.sh --builds" metadata: diff --git a/taskcluster/homebrew_tests-darwin-amd64.yml b/taskcluster/homebrew_tests-darwin-amd64.yml index dc93c183..bbf45fdc 100644 --- a/taskcluster/homebrew_tests-darwin-amd64.yml +++ b/taskcluster/homebrew_tests-darwin-amd64.yml @@ -4,6 +4,7 @@ build: artifact_url: ${system.homebrew_tests.url} artifact_namespace: ${system.homebrew_tests.namespace} scripts: + setup: "" build: "taskcluster/homebrew-build.sh --tests" package: "taskcluster/homebrew-package.sh --tests" metadata: diff --git a/taskcluster/host-build.sh b/taskcluster/host-build.sh index 1575832c..ddbc90fc 100755 --- a/taskcluster/host-build.sh +++ b/taskcluster/host-build.sh @@ -6,7 +6,7 @@ runtime=$1 source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index 50f78a2d..b49c94b0 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -4,6 +4,7 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_linux-amd64-cpu-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu" @@ -12,8 +13,9 @@ build: > ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.linux_amd64_cpu.url} scripts: + setup: "" build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" nc_asset_name: "native_client.amd64.cpu.linux.tar.xz" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml index f0979261..e3af5509 100644 --- a/taskcluster/linux-amd64-ctc-opt.yml +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -4,6 +4,7 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_linux-amd64-cpu-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.cpu-ctc" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.cpu-ctc" @@ -12,8 +13,9 @@ build: > ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.linux_amd64_cpu.url} scripts: + setup: "" build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' metadata: diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index f34f2de0..dc9e46be 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -4,6 +4,7 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_linux-amd64-gpu-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.gpu" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.gpu" @@ -12,9 +13,10 @@ build: > ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.gpu/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.linux_amd64_cuda.url} maxRunTime: 14400 scripts: + setup: "" build: "taskcluster/cuda-build.sh" package: "taskcluster/package.sh" nc_asset_name: "native_client.amd64.cuda.linux.tar.xz" diff --git a/taskcluster/linux-amd64-tflite-opt.yml b/taskcluster/linux-amd64-tflite-opt.yml index ffdb7499..10bcbc46 100644 --- a/taskcluster/linux-amd64-tflite-opt.yml +++ b/taskcluster/linux-amd64-tflite-opt.yml @@ -4,6 +4,7 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_linux-amd64-cpu-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.tflite" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.tflite" @@ -12,8 +13,9 @@ build: > ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.linux_amd64_cpu.url} scripts: + setup: "" build: "taskcluster/host-build.sh tflite" package: "taskcluster/package.sh" nc_asset_name: "native_client.amd64.tflite.linux.tar.xz" diff --git a/taskcluster/linux-arm64-cpu-opt.yml b/taskcluster/linux-arm64-cpu-opt.yml index aa4e2a34..0cfc3fae 100644 --- a/taskcluster/linux-arm64-cpu-opt.yml +++ b/taskcluster/linux-arm64-cpu-opt.yml @@ -4,6 +4,7 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_linux-arm64-cpu-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm64" @@ -19,8 +20,9 @@ build: system_config: > multistrap -d /tmp/multistrap-armbian64-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_armbian64_buster.conf - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.arm64/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.linux_arm64.url} scripts: + setup: "" build: "taskcluster/arm64-build.sh" package: "taskcluster/package.sh" nc_asset_name: "native_client.arm64.cpu.linux.tar.xz" diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index f98d84a6..88da6db1 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -44,9 +44,9 @@ then: extraSystemConfig: { $eval: strip(str(build.system_config)) } in: > adduser --system --home ${system.homedir.linux} ${system.username} && - apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} && + apt-get -qq update && apt-get -qq -y install ${deepspeech.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && cd ~/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 82959259..07293ea0 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -4,6 +4,7 @@ build: - "swig-linux-amd64" - "node-gyp-cache" - "pyenv-linux-amd64" + - "tf_linux-rpi3-cpu-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.arm" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.arm" @@ -19,8 +20,9 @@ build: system_config: > multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.arm/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.linux_armv7.url} scripts: + setup: "" build: "taskcluster/rpi3-build.sh" package: "taskcluster/package.sh" nc_asset_name: "native_client.rpi3.cpu.linux.tar.xz" diff --git a/taskcluster/node-build.sh b/taskcluster/node-build.sh index 8a898740..fcf68c04 100644 --- a/taskcluster/node-build.sh +++ b/taskcluster/node-build.sh @@ -6,6 +6,6 @@ package_option=$1 source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh do_deepspeech_npm_package "${package_option}" diff --git a/taskcluster/node-gyp-cache.yml b/taskcluster/node-gyp-cache.yml index a0c48ba5..e0e4d62f 100644 --- a/taskcluster/node-gyp-cache.yml +++ b/taskcluster/node-gyp-cache.yml @@ -8,6 +8,7 @@ build: > (apt-get -qq -y install sudo || true) scripts: + setup: "" build: "taskcluster/node-gyp-populate.sh" package: "taskcluster/node-gyp-package.sh" metadata: diff --git a/taskcluster/node-package-opt-base.tyml b/taskcluster/node-package-opt-base.tyml index b14cdd4c..a9989ca1 100644 --- a/taskcluster/node-package-opt-base.tyml +++ b/taskcluster/node-package-opt-base.tyml @@ -43,7 +43,7 @@ then: adduser --system --home ${system.homedir.linux} ${system.username} && apt-get -qq update && apt-get -qq -y install realpath git wget curl make sudo && ${extraSystemSetup} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ~/DeepSpeech/tf/ && touch ~/DeepSpeech/tf/tc-vars.sh && chmod +x ~/DeepSpeech/tf/tc-vars.sh && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} diff --git a/taskcluster/package.sh b/taskcluster/package.sh index 2acc737a..062654fd 100755 --- a/taskcluster/package.sh +++ b/taskcluster/package.sh @@ -6,7 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh mkdir -p ${TASKCLUSTER_ARTIFACTS} || true -cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ +cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ package_native_client "native_client.tar.xz" diff --git a/taskcluster/pyenv-darwin-amd64.yml b/taskcluster/pyenv-darwin-amd64.yml index 3ad055ca..eeebc414 100644 --- a/taskcluster/pyenv-darwin-amd64.yml +++ b/taskcluster/pyenv-darwin-amd64.yml @@ -4,6 +4,7 @@ build: artifact_url: ${system.pyenv.osx.url} artifact_namespace: ${system.pyenv.osx.namespace} scripts: + setup: "" build: "taskcluster/pyenv-build.sh" package: "taskcluster/pyenv-package.sh" metadata: diff --git a/taskcluster/pyenv-linux-amd64.yml b/taskcluster/pyenv-linux-amd64.yml index 4946e15b..25e1a27e 100644 --- a/taskcluster/pyenv-linux-amd64.yml +++ b/taskcluster/pyenv-linux-amd64.yml @@ -7,6 +7,7 @@ build: > apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_xenial.apt} wget scripts: + setup: "" build: "taskcluster/pyenv-build.sh" package: "taskcluster/pyenv-package.sh" metadata: diff --git a/taskcluster/pyenv-win-amd64.yml b/taskcluster/pyenv-win-amd64.yml index 7d05f141..1a19d9d4 100644 --- a/taskcluster/pyenv-win-amd64.yml +++ b/taskcluster/pyenv-win-amd64.yml @@ -4,6 +4,7 @@ build: artifact_url: "${system.pyenv.win.url}" artifact_namespace: "${system.pyenv.win.namespace}" scripts: + setup: "" build: "taskcluster/pyenv-build.sh" package: "taskcluster/pyenv-package.sh" metadata: diff --git a/taskcluster/rpi3-build.sh b/taskcluster/rpi3-build.sh index 2f1b29d3..e9b795a4 100755 --- a/taskcluster/rpi3-build.sh +++ b/taskcluster/rpi3-build.sh @@ -4,7 +4,7 @@ set -xe source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so diff --git a/taskcluster/swig-darwin-amd64.yml b/taskcluster/swig-darwin-amd64.yml index 64ca794b..c904c4a7 100644 --- a/taskcluster/swig-darwin-amd64.yml +++ b/taskcluster/swig-darwin-amd64.yml @@ -8,6 +8,7 @@ build: artifact_url: "${system.swig_build.osx.url}" artifact_namespace: "${system.swig_build.osx.namespace}" scripts: + setup: "" build: "taskcluster/build.sh" package: "taskcluster/package.sh" metadata: diff --git a/taskcluster/swig-linux-amd64.yml b/taskcluster/swig-linux-amd64.yml index 2a127bbc..d2e7bc0f 100644 --- a/taskcluster/swig-linux-amd64.yml +++ b/taskcluster/swig-linux-amd64.yml @@ -12,6 +12,7 @@ build: > apt-get -qq -y install autoconf automake bison build-essential scripts: + setup: "" build: "taskcluster/build.sh" package: "taskcluster/package.sh" metadata: diff --git a/taskcluster/swig-win-amd64.yml b/taskcluster/swig-win-amd64.yml index 310a8112..6648febd 100644 --- a/taskcluster/swig-win-amd64.yml +++ b/taskcluster/swig-win-amd64.yml @@ -13,6 +13,7 @@ build: apt-get -qq -y install autoconf automake bison build-essential mingw-w64 && (apt-get -qq -y install sudo || true) scripts: + setup: "" build: "taskcluster/build.sh x86_64-w64-mingw32" package: "taskcluster/package.sh" metadata: diff --git a/taskcluster/tc-all-utils.sh b/taskcluster/tc-all-utils.sh index 1ef5b325..769e9329 100755 --- a/taskcluster/tc-all-utils.sh +++ b/taskcluster/tc-all-utils.sh @@ -99,7 +99,7 @@ verify_bazel_rebuild() mkdir -p ${TASKCLUSTER_ARTIFACTS} || true - cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ + cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ spurious_rebuilds=$(grep 'Executing action' "${bazel_explain_file}" | grep 'Compiling' | grep -v -E 'no entry in the cache|unconditional execution is requested|Executing genrule //native_client:workspace_status|Compiling native_client/workspace_status.cc|Linking native_client/libdeepspeech.so' | wc -l) if [ "${spurious_rebuilds}" -ne 0 ]; then @@ -108,13 +108,13 @@ verify_bazel_rebuild() if is_patched_bazel; then mkdir -p ${DS_ROOT_TASK}/DeepSpeech/ckd/ds ${DS_ROOT_TASK}/DeepSpeech/ckd/tf tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ - tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ + tar xf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar --strip-components=4 -C ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ echo "Making a diff between CKD files" mkdir -p ${TASKCLUSTER_ARTIFACTS} - diff -urNw ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff + diff -urNw ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ | tee ${TASKCLUSTER_ARTIFACTS}/ckd.diff - rm -fr ${DS_ROOT_TASK}/DeepSpeech/ckd/tf/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ + rm -fr ${DS_ROOT_TASK}/DeepSpeech/ds/ckd/tensorflow/ ${DS_ROOT_TASK}/DeepSpeech/ckd/ds/ else echo "Cannot get CKD information from release, please use patched Bazel" fi; diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 2efc1dbc..0fae9d3c 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -49,7 +49,7 @@ export ANDROID_TMP_DIR=/data/local/tmp mkdir -p ${TASKCLUSTER_TMP_DIR} || true -export DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/tf +export DS_TFDIR=${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow export DS_DSDIR=${DS_ROOT_TASK}/DeepSpeech/ds export DS_EXAMPLEDIR=${DS_ROOT_TASK}/DeepSpeech/examples diff --git a/taskcluster/tc-build-utils.sh b/taskcluster/tc-build-utils.sh index 6a41a88a..f4042fbd 100755 --- a/taskcluster/tc-build-utils.sh +++ b/taskcluster/tc-build-utils.sh @@ -173,26 +173,26 @@ do_deepspeech_npm_package() do_bazel_build() { - cd ${DS_ROOT_TASK}/DeepSpeech/tf + cd ${DS_TFDIR} eval "export ${BAZEL_ENV_FLAGS}" if is_patched_bazel; then - find ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T - + find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-tf.tar -T - fi; bazel ${BAZEL_OUTPUT_USER_ROOT} build \ -s --explain bazel_monolithic.log --verbose_explanations --experimental_strict_action_env --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic -c opt ${BAZEL_BUILD_FLAGS} ${BAZEL_TARGETS} if is_patched_bazel; then - find ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T - + find ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-out/ -iname "*.ckd" | tar -cf ${DS_ROOT_TASK}/DeepSpeech/bazel-ckd-ds.tar -T - fi; - verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/tf/bazel_monolithic.log" + verify_bazel_rebuild "${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_monolithic.log" } shutdown_bazel() { - cd ${DS_ROOT_TASK}/DeepSpeech/tf + cd ${DS_TFDIR} bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown } diff --git a/taskcluster/tc-decision.py b/taskcluster/tc-decision.py new file mode 100644 index 00000000..a2324d84 --- /dev/null +++ b/taskcluster/tc-decision.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function, unicode_literals + +from glob import glob +from functools import reduce + +import json +import jsone +import os +import sys +import requests +import slugid +import yaml +import subprocess + +import networkx as nx + +TASKS_ROOT = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0]))) +TASKCLUSTER_API_BASEURL = 'http://taskcluster/queue/v1/task/%(task_id)s' + +def string_to_dict(sid, value): + parts = sid.split('.') + + def pack(parts): + if len(parts) == 1: + return {parts[0]: value} + elif len(parts): + return {parts[0]: pack(parts[1:])} + return parts + + return pack(parts) + +def merge_dicts(*dicts): + if not reduce(lambda x, y: isinstance(y, dict) and x, dicts, True): + raise TypeError("Object in *dicts not of type dict") + if len(dicts) < 2: + raise ValueError("Requires 2 or more dict objects") + + def merge(a, b): + for d in set(a.keys()).union(b.keys()): + if d in a and d in b: + if type(a[d]) == type(b[d]): + if not isinstance(a[d], dict): + ret = list({a[d], b[d]}) + if len(ret) == 1: ret = ret[0] + yield (d, sorted(ret)) + else: + yield (d, dict(merge(a[d], b[d]))) + else: + raise TypeError("Conflicting key:value type assignment", type(a[d]), a[d], type(b[d]), b[d]) + elif d in a: + yield (d, a[d]) + elif d in b: + yield (d, b[d]) + else: + raise KeyError + + return reduce(lambda x, y: dict(merge(x, y)), dicts[1:], dicts[0]) + +def taskcluster_event_context(): + das_context = {} + + # Pre-filterting + for k in os.environ.keys(): + if k == 'GITHUB_HEAD_USER': + os.environ['GITHUB_HEAD_USER_LOGIN'] = os.environ[k] + del os.environ['GITHUB_HEAD_USER'] + + for k in os.environ.keys(): + if k == 'TASK_ID': + parts = string_to_dict('taskcluster.taskGroupId', os.environ[k]) + das_context = merge_dicts(das_context, parts) + + if k.startswith('GITHUB_'): + parts = string_to_dict(k.lower().replace('_', '.').replace('github', 'event'), os.environ[k]) + das_context = merge_dicts(das_context, parts) + + return das_context + +def load_specific_contextFile(file): + specific_context = {} + + try: + with open(os.path.join(TASKS_ROOT, file)) as src: + specific_context = yaml.load(src) + + if specific_context is None: + specific_context = {} + except FileNotFoundError: + specific_context = {} + + return specific_context + +def defaultValues_build_context(): + return load_specific_contextFile('.build.yml') + +def shared_context(): + return load_specific_contextFile('.shared.yml') + +def create_task_payload(build, base_context): + print('build', build) + build_type = os.path.splitext(os.path.basename(build))[0] + + build_context = defaultValues_build_context() + with open(build) as src: + build_context['build'].update(yaml.load(src)['build']) + + # Be able to use what has been defined in base_context + # e.g., the {${event.head.branch}} + build_context = jsone.render(build_context, base_context) + template_context = { + 'taskcluster': { + 'taskId': as_slugid(build_type) + }, + 'build_type': build_type + } + + with open(os.path.join(TASKS_ROOT, build_context['build']['template_file'])) as src: + template = yaml.load(src) + + contextes = merge_dicts({}, base_context, template_context, build_context) + for one_context in glob(os.path.join(TASKS_ROOT, '*.cyml')): + with open(one_context) as src: + contextes = merge_dicts(contextes, yaml.load(src)) + + return jsone.render(template, contextes) + +def send_task(t): + url = TASKCLUSTER_API_BASEURL % {'task_id': t['taskId']} + del t['taskId'] + + r = requests.put(url, json=t) + + print(url, r.status_code) + if r.status_code != requests.codes.ok: + print(json.dumps(t, indent=2)) + print(r.content) + print(json.loads(r.content.decode())['message']) + + return r.status_code == requests.codes.ok + +slugids = {} +def as_slugid(name): + if name not in slugids: + slugids[name] = slugid.nice().decode() + print('cache miss', name, slugids[name]) + else: + print('cache hit', name, slugids[name]) + return slugids[name] + +def to_int(x): + return int(x) + +def functions_context(): + return { + 'as_slugid': as_slugid, + 'to_int': to_int + } + +def is_dry_run(): + return (len(sys.argv) > 1) and (sys.argv[1] == '--dry') + +def should_run(): + # Make a quick clone to fetch the last commit + try: + subprocess.check_call([ + 'git', 'clone', '--quiet', '-b', os.environ.get('GITHUB_HEAD_BRANCH'), + '--single-branch', os.environ.get('GITHUB_HEAD_REPO_URL'), + '--depth=1', '/tmp/ds-clone/' + ], env={'GIT_LFS_SKIP_SMUDGE': '1'}) + except subprocess.CalledProcessError as e: + print("Error while git cloning:", e, file=sys.stderr) + return False + + try: + git_msg = subprocess.check_output([ + 'git', '--git-dir=/tmp/ds-clone/.git/', + 'log', '--format=%b', '-n', '1', + os.environ.get('GITHUB_HEAD_SHA') + ]).decode('utf-8').strip().upper() + except subprocess.CalledProcessError as e: + print("Error while git show:", e, file=sys.stderr) + return False + + print('Commit message:', git_msg) + + x_deepspeech = filter(lambda x: 'X-DEEPSPEECH:' in x, git_msg.split('\n')) + if len(list(filter(lambda x: 'NOBUILD' in x, x_deepspeech))) == 1: + print('Not running anything according to commit message') + return False + + return True + +if __name__ == '__main__': + if not is_dry_run(): + # We might want to NOT run in some cases + if not should_run(): + sys.exit(0) + + base_context = taskcluster_event_context() + base_context = merge_dicts(base_context, functions_context()) + base_context = merge_dicts(base_context, shared_context()) + + root_task = base_context['taskcluster']['taskGroupId'] + + tasks_graph = nx.DiGraph() + tasks = {} + + for build in glob(os.path.join(TASKS_ROOT, '*.yml')): + t = create_task_payload(build, base_context) + + # We allow template to produce completely empty output + if not t: + continue + + if 'dependencies' in t and len(t['dependencies']) > 0: + for dep in t['dependencies']: + tasks_graph.add_edge(t['taskId'], dep) + else: + tasks_graph.add_edge(t['taskId'], root_task) + + tasks[t['taskId']] = t + + for task in nx.dfs_postorder_nodes(tasks_graph): + # root_task is the task group and also the task id that is already + # running, so we don't have to schedule that + if task == root_task: + continue + + t = tasks[task] + if is_dry_run(): + print(json.dumps(t, indent=2)) + continue + + p = send_task(t) + if not p: + sys.exit(1) diff --git a/taskcluster/tc-decision_reqs.txt b/taskcluster/tc-decision_reqs.txt new file mode 100644 index 00000000..230c187a --- /dev/null +++ b/taskcluster/tc-decision_reqs.txt @@ -0,0 +1,5 @@ +json-e == 2.3.1 +networkx +pyaml +requests +slugid == 1.0.7 diff --git a/taskcluster/tc-schedule.sh b/taskcluster/tc-schedule.sh index a43a69c9..5b348041 100755 --- a/taskcluster/tc-schedule.sh +++ b/taskcluster/tc-schedule.sh @@ -2,15 +2,13 @@ set -ex -# tc-decision.py assumes being at the root folder -curdir=$(dirname "$0")/.. +curdir=$(dirname "$0")/ pip3 install --quiet --user --upgrade pip export PATH=$HOME/.local/bin/:$PATH -curl -L --silent https://raw.githubusercontent.com/lissyx/taskcluster-github-decision/${TC_DECISION_SHA}/requirements.txt | pip3 install --quiet --user --upgrade -r /dev/stdin -curl -L --silent https://raw.githubusercontent.com/lissyx/taskcluster-github-decision/${TC_DECISION_SHA}/tc-decision.py > ${curdir}/tc-decision.py +pip3 install --quiet --user --upgrade -r ${curdir}/tc-decision_reqs.txt # First, perform dry run for push and pull request # This should help us track merge failures in advance diff --git a/taskcluster/test-win-cuda-opt-base.tyml b/taskcluster/test-win-cuda-opt-base.tyml index 9fa37404..17ee34ae 100644 --- a/taskcluster/test-win-cuda-opt-base.tyml +++ b/taskcluster/test-win-cuda-opt-base.tyml @@ -38,8 +38,6 @@ then: "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0" - - .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}" - - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - $let: extraSystemSetup: { $eval: strip(str(build.system_setup)) } @@ -69,10 +67,6 @@ then: content: sha256: ${system.msys2.sha} url: ${system.msys2.url} - - file: filesystem-2020.02-3-x86_64.pkg.tar.xz - content: - sha256: ${system.msys2_filesystem_pkg.sha} - url: ${system.msys2_filesystem_pkg.url} - file: pyenv.tar.gz content: url: ${system.pyenv.win.url} diff --git a/taskcluster/test-win-opt-base.tyml b/taskcluster/test-win-opt-base.tyml index f8925d6d..92021d74 100644 --- a/taskcluster/test-win-opt-base.tyml +++ b/taskcluster/test-win-opt-base.tyml @@ -40,8 +40,6 @@ then: "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0" - - .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}" - - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - $let: extraSystemSetup: { $eval: strip(str(build.system_setup)) } @@ -71,10 +69,6 @@ then: content: sha256: ${system.msys2.sha} url: ${system.msys2.url} - - file: filesystem-2020.02-3-x86_64.pkg.tar.xz - content: - sha256: ${system.msys2_filesystem_pkg.sha} - url: ${system.msys2_filesystem_pkg.url} - file: pyenv.tar.gz content: url: ${system.pyenv.win.url} diff --git a/taskcluster/tf_android-arm64-opt.yml b/taskcluster/tf_android-arm64-opt.yml new file mode 100644 index 00000000..1b9d93ca --- /dev/null +++ b/taskcluster/tf_android-arm64-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.android_arm64.url} + artifact_namespace: ${system.tensorflow.android_arm64.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh --android" + build: "taskcluster/tf_tc-build.sh --android-arm64" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + metadata: + name: "TensorFlow Android ARM64" + description: "Building TensorFlow for Android ARM64, optimized version" diff --git a/taskcluster/tf_android-armv7-opt.yml b/taskcluster/tf_android-armv7-opt.yml new file mode 100644 index 00000000..ec28464c --- /dev/null +++ b/taskcluster/tf_android-armv7-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.android_armv7.url} + artifact_namespace: ${system.tensorflow.android_armv7.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh --android" + build: "taskcluster/tf_tc-build.sh --android-armv7" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + metadata: + name: "TensorFlow Android ARMv7" + description: "Building TensorFlow for Android ARMv7, optimized version" diff --git a/taskcluster/tf_darwin-amd64-opt.yml b/taskcluster/tf_darwin-amd64-opt.yml new file mode 100644 index 00000000..21c19021 --- /dev/null +++ b/taskcluster/tf_darwin-amd64-opt.yml @@ -0,0 +1,18 @@ +build: + template_file: generic_tc_caching-darwin-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.darwin_amd64.url} + artifact_namespace: ${system.tensorflow.darwin_amd64.namespace} + generic: + workerType: "ds-macos-heavy" + system_config: + > + ${tensorflow.packages_macos.brew} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --osx" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 28800 + metadata: + name: "TensorFlow OSX AMD64 CPU" + description: "Building TensorFlow for OSX AMD64, CPU only, optimized version" diff --git a/taskcluster/tf_linux-amd64-cpu-opt.yml b/taskcluster/tf_linux-amd64-cpu-opt.yml new file mode 100644 index 00000000..d869d280 --- /dev/null +++ b/taskcluster/tf_linux-amd64-cpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.linux_amd64_cpu.url} + artifact_namespace: ${system.tensorflow.linux_amd64_cpu.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + metadata: + name: "TensorFlow Linux AMD64 CPU" + description: "Building TensorFlow for Linux/AMD64, CPU only, optimized version" diff --git a/taskcluster/tf_linux-amd64-gpu-opt.yml b/taskcluster/tf_linux-amd64-gpu-opt.yml new file mode 100644 index 00000000..38e94808 --- /dev/null +++ b/taskcluster/tf_linux-amd64-gpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.linux_amd64_cuda.url} + artifact_namespace: ${system.tensorflow.linux_amd64_cuda.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh --cuda" + build: "taskcluster/tf_tc-build.sh --gpu" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + metadata: + name: "TensorFlow Linux AMD64 CUDA" + description: "Building TensorFlow for Linux/AMD64, CUDA-enabled, optimized version" diff --git a/taskcluster/tf_linux-arm64-cpu-opt.yml b/taskcluster/tf_linux-arm64-cpu-opt.yml new file mode 100644 index 00000000..a17b3629 --- /dev/null +++ b/taskcluster/tf_linux-arm64-cpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.linux_arm64.url} + artifact_namespace: ${system.tensorflow.linux_arm64.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --arm64" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + metadata: + name: "TensorFlow Linux ARM64 Cortex-A53 CPU" + description: "Building TensorFlow for Linux ARM64 Cortex-A53, CPU only, optimized version" diff --git a/taskcluster/tf_linux-rpi3-cpu-opt.yml b/taskcluster/tf_linux-rpi3-cpu-opt.yml new file mode 100644 index 00000000..579c42c9 --- /dev/null +++ b/taskcluster/tf_linux-rpi3-cpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-linux-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.linux_armv7.url} + artifact_namespace: ${system.tensorflow.linux_armv7.namespace} + system_config: + > + ${tensorflow.packages_xenial.apt} && ${java.packages_xenial.apt} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --arm" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + metadata: + name: "TensorFlow Linux RPi3/ARMv7 CPU" + description: "Building TensorFlow for Linux RPi3 ARMv7, CPU only, optimized version" diff --git a/taskcluster/tf_tc-brew.sh b/taskcluster/tf_tc-brew.sh new file mode 100755 index 00000000..297b2bba --- /dev/null +++ b/taskcluster/tf_tc-brew.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -ex + +if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then + echo "No TASKCLUSTER_TASK_DIR, aborting." + exit 1 +fi + +LOCAL_BREW="${TASKCLUSTER_TASK_DIR}/homebrew" +export PATH=${LOCAL_BREW}/bin:$PATH +export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/homebrew.logs/" +export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/homebrew.cache/" + +# Never fail on pre-existing homebrew/ directory +mkdir -p "${LOCAL_BREW}" || true +mkdir -p "${HOMEBREW_CACHE}" || true + +# Make sure to verify there is a 'brew' binary there, otherwise install things. +if [ ! -x "${LOCAL_BREW}/bin/brew" ]; then + curl -L https://github.com/Homebrew/brew/tarball/2.2.17 | tar xz --strip 1 -C "${LOCAL_BREW}" +fi; + +echo "local brew list (should be empty) ..." +brew list + +echo "local brew prefix ..." +local_prefix=$(brew --prefix) +echo "${local_prefix}" + +if [ "${LOCAL_BREW}" != "${local_prefix}" ]; then + echo "Weird state:" + echo "LOCAL_BREW=${LOCAL_BREW}" + echo "local_prefix=${local_prefix}" + exit 1 +fi; + +# coreutils, pyenv-virtualenv required for build of tensorflow +all_pkgs="coreutils pyenv-virtualenv" + +for pkg in ${all_pkgs}; +do + (brew list --versions ${pkg} && brew upgrade ${pkg}) || brew install ${pkg} +done; diff --git a/taskcluster/tf_tc-build.sh b/taskcluster/tf_tc-build.sh new file mode 100755 index 00000000..ad8085ee --- /dev/null +++ b/taskcluster/tf_tc-build.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +set -ex + +source $(dirname $0)/tf_tc-vars.sh + +build_amd64=yes +build_gpu=no +build_android_arm=no +build_android_arm64=no +build_linux_arm=no +build_linux_arm64=no + +if [ "$1" = "--gpu" ]; then + build_amd64=yes + build_gpu=yes + build_android_arm=no + build_android_arm64=no + build_linux_arm=no + build_linux_arm64=no +fi + +if [ "$1" = "--arm" ]; then + build_amd64=yes + build_gpu=no + build_android_arm=no + build_android_arm64=no + build_linux_arm=yes + build_linux_arm64=no +fi + +if [ "$1" = "--arm64" ]; then + build_amd64=yes + build_gpu=no + build_android_arm=no + build_android_arm64=no + build_linux_arm=no + build_linux_arm64=yes +fi + +if [ "$1" = "--android-armv7" ]; then + build_amd64=no + build_gpu=no + build_android_arm=yes + build_android_arm64=no + build_linux_arm=no + build_linux_arm64=no +fi + +if [ "$1" = "--android-arm64" ]; then + build_amd64=no + build_gpu=no + build_android_arm=no + build_android_arm64=yes + build_linux_arm=no + build_linux_arm64=no +fi + +pushd ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/ + BAZEL_BUILD="bazel ${BAZEL_OUTPUT_USER_ROOT} build -s --explain bazel_monolithic_tf.log --verbose_explanations --experimental_strict_action_env --config=monolithic" + + # Start a bazel process to ensure reliability on Windows and avoid: + # FATAL: corrupt installation: file 'c:\builds\tc-workdir\.bazel_cache/install/6b1660721930e9d5f231f7d2a626209b/_embedded_binaries/build-runfiles.exe' missing. + bazel ${BAZEL_OUTPUT_USER_ROOT} info + + # Force toolchain sync (useful on macOS ?) + bazel ${BAZEL_OUTPUT_USER_ROOT} sync --configure + + if [ "${build_amd64}" = "yes" ]; then + # Pure amd64 CPU-only build + if [ "${OS}" = "${TC_MSYS_VERSION}" -a "${build_gpu}" = "no" ]; then + echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB} + elif [ "${build_gpu}" = "no" -a "${build_linux_arm}" = "no" -a "${build_linux_arm64}" = "no" ]; then + echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_OPT_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LIB_CPP_API} ${BUILD_TARGET_LITE_LIB} + fi + + # Cross RPi3 CPU-only build + if [ "${build_linux_arm}" = "yes" ]; then + echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + fi + + # Cross ARM64 Cortex-A53 build + if [ "${build_linux_arm64}" = "yes" ]; then + echo "" | TF_NEED_CUDA=0 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + fi + + # Pure amd64 GPU-enabled build + if [ "${build_gpu}" = "yes" ]; then + eval "export ${TF_CUDA_FLAGS}" && (echo "" | TF_NEED_CUDA=1 ./configure) && ${BAZEL_BUILD} -c opt ${BAZEL_CUDA_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BAZEL_OPT_FLAGS} ${BUILD_TARGET_LIB_CPP_API} + fi + fi + + if [ "${build_android_arm}" = "yes" ]; then + echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ANDROID_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + fi; + + if [ "${build_android_arm64}" = "yes" ]; then + echo "" | TF_SET_ANDROID_WORKSPACE=1 ./configure && ${BAZEL_BUILD} -c opt ${BAZEL_ANDROID_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS} ${BUILD_TARGET_LITE_LIB} + fi; + + if [ $? -ne 0 ]; then + # There was a failure, just account for it. + echo "Build failure, please check the output above. Exit code was: $?" + return 1 + fi + + bazel ${BAZEL_OUTPUT_USER_ROOT} shutdown +popd diff --git a/taskcluster/tf_tc-package.sh b/taskcluster/tf_tc-package.sh new file mode 100755 index 00000000..1291dce1 --- /dev/null +++ b/taskcluster/tf_tc-package.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -xe + +source $(dirname $0)/tf_tc-vars.sh + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel_*.log ${TASKCLUSTER_ARTIFACTS} + +OUTPUT_ROOT="${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin" + +for output_bin in \ + tensorflow/lite/experimental/c/libtensorflowlite_c.so \ + tensorflow/tools/graph_transforms/transform_graph \ + tensorflow/tools/graph_transforms/summarize_graph \ + tensorflow/tools/benchmark/benchmark_model \ + tensorflow/contrib/util/convert_graphdef_memmapped_format \ + tensorflow/lite/toco/toco; +do + if [ -f "${OUTPUT_ROOT}/${output_bin}" ]; then + cp ${OUTPUT_ROOT}/${output_bin} ${TASKCLUSTER_ARTIFACTS}/ + fi; +done; + +if [ -f "${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model" ]; then + cp ${OUTPUT_ROOT}/tensorflow/lite/tools/benchmark/benchmark_model ${TASKCLUSTER_ARTIFACTS}/lite_benchmark_model +fi + +# It seems that bsdtar and gnutar are behaving a bit differently on the way +# they deal with --exclude="./public/*" ; this caused ./DeepSpeech/tensorflow/core/public/ +# to be ditched when we just wanted to get rid of ./public/ on OSX. +# Switching to gnutar (already needed for the --transform on DeepSpeech tasks) +# does the trick. +TAR_EXCLUDE="--exclude=./dls/*" +if [ "${OS}" = "Darwin" ]; then + TAR_EXCLUDE="--exclude=./dls/* --exclude=./public/* --exclude=./generic-worker/* --exclude=./homebrew/* --exclude=./homebrew.cache/* --exclude=./homebrew.logs/*" +fi; + +# Make a tar of +# - /home/build-user/ (linux +# - /Users/build-user/TaskCluster/HeavyTasks/X/ (OSX) +# - C:\builds\tc-workdir\ (windows) + +if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + export PATH=$PATH:'/c/Program Files/7-Zip/' + pushd ${DS_ROOT_TASK} + 7z a '-xr!.\dls\' '-xr!.\tmp\' '-xr!.\msys64\' -snl -snh -so home.tar . | 7z a -si ${TASKCLUSTER_ARTIFACTS}/home.tar.xz + popd +else + ${TAR} -C ${DS_ROOT_TASK} ${TAR_EXCLUDE} -cf - . | ${XZ} > ${TASKCLUSTER_ARTIFACTS}/home.tar.xz +fi + +if [ "${OS}" = "Linux" ]; then + SHA_SUM_GEN="sha256sum" +elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + SHA_SUM_GEN="sha256sum" +elif [ "${OS}" = "Darwin" ]; then + SHA_SUM_GEN="shasum -a 256" +fi; + +${SHA_SUM_GEN} ${TASKCLUSTER_ARTIFACTS}/* > ${TASKCLUSTER_ARTIFACTS}/checksums.txt diff --git a/taskcluster/tf_tc-pip.sh b/taskcluster/tf_tc-pip.sh new file mode 100755 index 00000000..9e16c445 --- /dev/null +++ b/taskcluster/tf_tc-pip.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -ex + +# Taken from https://www.tensorflow.org/install/source +# Only future is needed for our builds, as we don't build the Python package +pip install -U --user future==0.17.1 diff --git a/taskcluster/tf_tc-setup.sh b/taskcluster/tf_tc-setup.sh new file mode 100755 index 00000000..3e5973e0 --- /dev/null +++ b/taskcluster/tf_tc-setup.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +set -ex + +source $(dirname $0)/tf_tc-vars.sh + +install_cuda= +if [ "$1" = "--cuda" ]; then + install_cuda=yes +fi + +install_android= +if [ "$1" = "--android" ]; then + install_android=yes +fi + +# $1 url +# $2 sha256 +download() +{ + fname=`basename $1` + + ${WGET} $1 -O ${DS_ROOT_TASK}/dls/$fname && echo "$2 ${DS_ROOT_TASK}/dls/$fname" | ${SHA_SUM} - +} + +# Download stuff +mkdir -p ${DS_ROOT_TASK}/dls || true +download $BAZEL_URL $BAZEL_SHA256 + +if [ ! -z "${install_cuda}" ]; then + download $CUDA_URL $CUDA_SHA256 + download $CUDNN_URL $CUDNN_SHA256 +fi; + +if [ ! -z "${install_android}" ]; then + download $ANDROID_NDK_URL $ANDROID_NDK_SHA256 + download $ANDROID_SDK_URL $ANDROID_SDK_SHA256 +fi; + +# For debug +ls -hal ${DS_ROOT_TASK}/dls/ + +# Install Bazel in ${DS_ROOT_TASK}/bin +BAZEL_INSTALL_FILENAME=$(basename "${BAZEL_URL}") +if [ "${OS}" = "Linux" ]; then + BAZEL_INSTALL_FLAGS="--user" +elif [ "${OS}" = "Darwin" ]; then + BAZEL_INSTALL_FLAGS="--bin=${DS_ROOT_TASK}/bin --base=${DS_ROOT_TASK}/.bazel" +fi; +mkdir -p ${DS_ROOT_TASK}/bin || true +pushd ${DS_ROOT_TASK}/bin + if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + cp ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${DS_ROOT_TASK}/bin/bazel.exe + else + /bin/bash ${DS_ROOT_TASK}/dls/${BAZEL_INSTALL_FILENAME} ${BAZEL_INSTALL_FLAGS} + fi +popd + +# For debug +bazel version + +bazel shutdown + +if [ ! -z "${install_cuda}" ]; then + # Install CUDA and CuDNN + mkdir -p ${DS_ROOT_TASK}/DeepSpeech/CUDA/ || true + pushd ${DS_ROOT_TASK} + CUDA_FILE=`basename ${CUDA_URL}` + PERL5LIB=. sh ${DS_ROOT_TASK}/dls/${CUDA_FILE} --silent --override --toolkit --toolkitpath=${DS_ROOT_TASK}/DeepSpeech/CUDA/ --defaultroot=${DS_ROOT_TASK}/DeepSpeech/CUDA/ + + CUDNN_FILE=`basename ${CUDNN_URL}` + tar xvf ${DS_ROOT_TASK}/dls/${CUDNN_FILE} --strip-components=1 -C ${DS_ROOT_TASK}/DeepSpeech/CUDA/ + popd + + LD_LIBRARY_PATH=${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/:${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/:$LD_LIBRARY_PATH + export LD_LIBRARY_PATH + + # We might lack libcuda.so.1 symlink, let's fix as upstream does: + # https://github.com/tensorflow/tensorflow/pull/13811/files?diff=split#diff-2352449eb75e66016e97a591d3f0f43dR96 + if [ ! -h "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so.1" ]; then + ln -s "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so" "${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/libcuda.so.1" + fi; + +else + echo "No CUDA/CuDNN to install" +fi + +if [ ! -z "${install_android}" ]; then + mkdir -p ${DS_ROOT_TASK}/DeepSpeech/Android/SDK || true + ANDROID_NDK_FILE=`basename ${ANDROID_NDK_URL}` + ANDROID_SDK_FILE=`basename ${ANDROID_SDK_URL}` + + pushd ${DS_ROOT_TASK}/DeepSpeech/Android + unzip ${DS_ROOT_TASK}/dls/${ANDROID_NDK_FILE} + popd + + pushd ${DS_ROOT_TASK}/DeepSpeech/Android/SDK + unzip ${DS_ROOT_TASK}/dls/${ANDROID_SDK_FILE} + yes | ./tools/bin/sdkmanager --licenses + ./tools/bin/sdkmanager --update + ./tools/bin/sdkmanager --install "platforms;android-16" "build-tools;28.0.3" + popd +fi + +mkdir -p ${TASKCLUSTER_ARTIFACTS} || true + +pushd ${DS_ROOT_TASK}/DeepSpeech/ds/ + git submodule --quiet sync tensorflow/ && git submodule --quiet update --init tensorflow/ +popd + + +# Taken from https://www.tensorflow.org/install/source +# Only future is needed for our builds, as we don't build the Python package +pip install -U --user future==0.17.1 || true diff --git a/taskcluster/tf_tc-vars.sh b/taskcluster/tf_tc-vars.sh new file mode 100755 index 00000000..c6aa1746 --- /dev/null +++ b/taskcluster/tf_tc-vars.sh @@ -0,0 +1,195 @@ +#!/bin/bash + +set -ex + +export OS=$(uname) +if [ "${OS}" = "Linux" ]; then + export DS_ROOT_TASK=$(/usr/bin/realpath "${HOME}") + + BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-linux-x86_64.sh + BAZEL_SHA256=2fbdc9c0e3d376697caf0ee3673b7c9475214068c55a01b9744891e131f90b87 + + CUDA_URL=http://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run + CUDA_SHA256=e7c22dc21278eb1b82f34a60ad7640b41ad3943d929bebda3008b72536855d31 + + # From https://gitlab.com/nvidia/cuda/blob/centos7/10.1/devel/cudnn7/Dockerfile + CUDNN_URL=http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.0/cudnn-10.1-linux-x64-v7.6.0.64.tgz + CUDNN_SHA256=e956c6f9222fcb867a10449cfc76dee5cfd7c7531021d95fe9586d7e043b57d7 + + ANDROID_NDK_URL=https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip + ANDROID_NDK_SHA256=4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd + + ANDROID_SDK_URL=https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip + ANDROID_SDK_SHA256=92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9 + + SHA_SUM="sha256sum -c --strict" + WGET=/usr/bin/wget + TAR=tar + XZ="pixz -9" +elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + if [ -z "${TASKCLUSTER_TASK_DIR}" -o -z "${TASKCLUSTER_ARTIFACTS}" ]; then + echo "Inconsistent Windows setup: missing some vars." + echo "TASKCLUSTER_TASK_DIR=${TASKCLUSTER_TASK_DIR}" + echo "TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS}" + exit 1 + fi; + + # Re-export with cygpath to make sure it is sane, otherwise it might trigger + # unobvious failures with cp etc. + export TASKCLUSTER_TASK_DIR="$(cygpath ${TASKCLUSTER_TASK_DIR})" + export TASKCLUSTER_ARTIFACTS="$(cygpath ${TASKCLUSTER_ARTIFACTS})" + + export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} + export BAZEL_VC='C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC' + export BAZEL_SH='C:\builds\tc-workdir\msys64\usr\bin\bash' + export TC_WIN_BUILD_PATH='C:\builds\tc-workdir\msys64\usr\bin;C:\Python36' + export MSYS2_ARG_CONV_EXCL='//' + + mkdir -p ${TASKCLUSTER_TASK_DIR}/tmp/ + export TEMP=${TASKCLUSTER_TASK_DIR}/tmp/ + export TMP=${TASKCLUSTER_TASK_DIR}/tmp/ + + BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-windows-x86_64.exe + BAZEL_SHA256=cc7b3ff6f4bfd6bc2121a80656afec66ee57713e8b88e9d2fb58b4eddf271268 + + CUDA_INSTALL_DIRECTORY=$(cygpath 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1') + + SHA_SUM="sha256sum -c --strict" + WGET=wget + TAR=/usr/bin/tar.exe + XZ="xz -9 -T0" +elif [ "${OS}" = "Darwin" ]; then + if [ -z "${TASKCLUSTER_TASK_DIR}" -o -z "${TASKCLUSTER_ARTIFACTS}" ]; then + echo "Inconsistent OSX setup: missing some vars." + echo "TASKCLUSTER_TASK_DIR=${TASKCLUSTER_TASK_DIR}" + echo "TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS}" + exit 1 + fi; + + export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} + + BAZEL_URL=https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel-2.0.0-installer-darwin-x86_64.sh + BAZEL_SHA256=c675fa27d99a3114d681db10eb03ded547c40f702b2048c99b8f4ea8e89b9356 + + SHA_SUM="shasum -a 256 -c" + WGET=wget + TAR=gtar + XZ="pixz -9" +fi; + +# /tmp/artifacts for docker-worker on linux, +# and task subdir for generic-worker on osx +export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts} + +### Define variables that needs to be exported to other processes + +PATH=${DS_ROOT_TASK}/bin:$PATH +if [ "${OS}" = "Darwin" ]; then + PATH=${DS_ROOT_TASK}/homebrew/bin/:${DS_ROOT_TASK}/homebrew/opt/node@10/bin:$PATH +fi; +export PATH + +if [ "${OS}" = "Linux" ]; then + export LD_LIBRARY_PATH=${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/:${DS_ROOT_TASK}/DeepSpeech/CUDA/lib64/stubs/:$LD_LIBRARY_PATH + export ANDROID_SDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/SDK/ + export ANDROID_NDK_HOME=${DS_ROOT_TASK}/DeepSpeech/Android/android-ndk-r18b/ +fi; + +export TF_ENABLE_XLA=0 +if [ "${OS}" = "Linux" ]; then + TF_NEED_JEMALLOC=1 +elif [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + TF_NEED_JEMALLOC=0 +elif [ "${OS}" = "Darwin" ]; then + TF_NEED_JEMALLOC=0 +fi; +export TF_NEED_JEMALLOC +export TF_NEED_OPENCL_SYCL=0 +export TF_NEED_MKL=0 +export TF_NEED_VERBS=0 +export TF_NEED_MPI=0 +export TF_NEED_IGNITE=0 +export TF_NEED_GDR=0 +export TF_NEED_NGRAPH=0 +export TF_DOWNLOAD_CLANG=0 +export TF_SET_ANDROID_WORKSPACE=0 +export TF_NEED_TENSORRT=0 +export TF_NEED_ROCM=0 + +# This should be gcc-5, hopefully. CUDA and TensorFlow might not be happy, otherwise. +export GCC_HOST_COMPILER_PATH=/usr/bin/gcc + +if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + export PYTHON_BIN_PATH=C:/Python36/python.exe +else + export PYTHON_BIN_PATH=/usr/bin/python2.7 +fi + +## Below, define or export some build variables + +# Enable some SIMD support. Limit ourselves to what Tensorflow needs. +# Also ensure to not require too recent CPU: AVX2/FMA introduced by: +# - Intel with Haswell (2013) +# - AMD with Excavator (2015) +# For better compatibility, AVX ony might be better. +# +# Build for generic amd64 platforms, no device-specific optimization +# See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for targetting specific CPUs + +if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + CC_OPT_FLAGS="/arch:AVX" +else + CC_OPT_FLAGS="-mtune=generic -march=x86-64 -msse -msse2 -msse3 -msse4.1 -msse4.2 -mavx" +fi +BAZEL_OPT_FLAGS="" +for flag in ${CC_OPT_FLAGS}; +do + BAZEL_OPT_FLAGS="${BAZEL_OPT_FLAGS} --copt=${flag}" +done; + +export CC_OPT_FLAGS + +BAZEL_OUTPUT_CACHE_DIR="${DS_ROOT_TASK}/.bazel_cache/" +BAZEL_OUTPUT_CACHE_INSTANCE="${BAZEL_OUTPUT_CACHE_DIR}/output/" +mkdir -p ${BAZEL_OUTPUT_CACHE_INSTANCE} || true + +# We need both to ensure stable path ; default value for output_base is some +# MD5 value. +BAZEL_OUTPUT_USER_ROOT="--output_user_root ${BAZEL_OUTPUT_CACHE_DIR} --output_base ${BAZEL_OUTPUT_CACHE_INSTANCE}" +export BAZEL_OUTPUT_USER_ROOT + +NVCC_COMPUTE="3.5" + +### Define build parameters/env variables that we will re-ues in sourcing scripts. +if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_PATHS=\"${CUDA_INSTALL_DIRECTORY}\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\"" +else + TF_CUDA_FLAGS="TF_CUDA_CLANG=0 TF_CUDA_VERSION=10.1 TF_CUDNN_VERSION=7.6.0 CUDNN_INSTALL_PATH=\"${DS_ROOT_TASK}/DeepSpeech/CUDA\" TF_CUDA_PATHS=\"${DS_ROOT_TASK}/DeepSpeech/CUDA\" TF_CUDA_COMPUTE_CAPABILITIES=\"${NVCC_COMPUTE}\"" +fi +BAZEL_ARM_FLAGS="--config=rpi3 --config=rpi3_opt --copt=-DTFLITE_WITH_RUY_GEMV" +BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_WITH_RUY_GEMV" +BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" +BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" +BAZEL_CUDA_FLAGS="--config=cuda" + +if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then + # Somehow, even with Python being in the PATH, Bazel on windows struggles + # with '/usr/bin/env python' ... + # + # We also force TMP/TEMP otherwise Bazel will pick default Windows one + # under %USERPROFILE%\AppData\Local\Temp and with 8.3 file format convention + # it messes with cxx_builtin_include_directory + BAZEL_EXTRA_FLAGS="--action_env=PATH=${TC_WIN_BUILD_PATH} --action_env=TEMP=${TEMP} --action_env=TMP=${TMP}" +else + BAZEL_EXTRA_FLAGS="--config=noaws --config=nogcp --config=nohdfs --config=nonccl --copt=-fvisibility=hidden" +fi + +### Define build targets that we will re-ues in sourcing scripts. +BUILD_TARGET_LIB_CPP_API="//tensorflow:tensorflow_cc" +BUILD_TARGET_GRAPH_TRANSFORMS="//tensorflow/tools/graph_transforms:transform_graph" +BUILD_TARGET_GRAPH_SUMMARIZE="//tensorflow/tools/graph_transforms:summarize_graph" +BUILD_TARGET_GRAPH_BENCHMARK="//tensorflow/tools/benchmark:benchmark_model" +#BUILD_TARGET_CONVERT_MMAP="//tensorflow/contrib/util:convert_graphdef_memmapped_format" +BUILD_TARGET_TOCO="//tensorflow/lite/toco:toco" +BUILD_TARGET_LITE_BENCHMARK="//tensorflow/lite/tools/benchmark:benchmark_model" +BUILD_TARGET_LITE_LIB="//tensorflow/lite/c:libtensorflowlite_c.so" diff --git a/taskcluster/tf_win-amd64-cpu-opt.yml b/taskcluster/tf_win-amd64-cpu-opt.yml new file mode 100644 index 00000000..4b4d1105 --- /dev/null +++ b/taskcluster/tf_win-amd64-cpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-win-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.win_amd64_cpu.url} + artifact_namespace: ${system.tensorflow.win_amd64_cpu.namespace} + system_config: + > + ${tensorflow.packages_win.pacman} && ${tensorflow.packages_win.msys64} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + metadata: + name: "TensorFlow Windows AMD64 CPU" + description: "Building TensorFlow for Windows AMD64, CPU only, optimized version" diff --git a/taskcluster/tf_win-amd64-gpu-opt.yml b/taskcluster/tf_win-amd64-gpu-opt.yml new file mode 100644 index 00000000..63fa1ac5 --- /dev/null +++ b/taskcluster/tf_win-amd64-gpu-opt.yml @@ -0,0 +1,16 @@ +build: + template_file: generic_tc_caching-win-opt-base.tyml + cache: + artifact_url: ${system.tensorflow.win_amd64_cuda.url} + artifact_namespace: ${system.tensorflow.win_amd64_cuda.namespace} + system_config: + > + ${tensorflow.packages_win.pacman} && ${tensorflow.packages_win.msys64} + scripts: + setup: "taskcluster/tf_tc-setup.sh" + build: "taskcluster/tf_tc-build.sh --gpu" + package: "taskcluster/tf_tc-package.sh" + maxRunTime: 14400 + metadata: + name: "TensorFlow Windows AMD64 CUDA" + description: "Building TensorFlow for Windows AMD64, CUDA, optimized version" diff --git a/taskcluster/win-amd64-cpu-opt.yml b/taskcluster/win-amd64-cpu-opt.yml index b7dbba5b..d74a5c6d 100644 --- a/taskcluster/win-amd64-cpu-opt.yml +++ b/taskcluster/win-amd64-cpu-opt.yml @@ -8,7 +8,8 @@ build: - "node-gyp-cache" - "swig-win-amd64" - "pyenv-win-amd64" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz" + - "tf_win-amd64-cpu-opt" + tensorflow: ${system.tensorflow.win_amd64_cpu.url} scripts: build: "taskcluster/win-build.sh" package: "taskcluster/win-package.sh" diff --git a/taskcluster/win-amd64-ctc-opt.yml b/taskcluster/win-amd64-ctc-opt.yml index b17bc53c..0b30f581 100644 --- a/taskcluster/win-amd64-ctc-opt.yml +++ b/taskcluster/win-amd64-ctc-opt.yml @@ -4,11 +4,12 @@ build: - "swig-win-amd64" - "node-gyp-cache" - "pyenv-win-amd64" + - "tf_win-amd64-cpu-opt" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win-ctc" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win-ctc" - "index.project.deepspeech.deepspeech.native_client.win-ctc.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz" + tensorflow: ${system.tensorflow.win_amd64_cpu.url} scripts: build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' diff --git a/taskcluster/win-amd64-gpu-opt.yml b/taskcluster/win-amd64-gpu-opt.yml index b9c99395..e3dd5747 100644 --- a/taskcluster/win-amd64-gpu-opt.yml +++ b/taskcluster/win-amd64-gpu-opt.yml @@ -8,7 +8,8 @@ build: - "node-gyp-cache" - "swig-win-amd64" - "pyenv-win-amd64" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win-cuda/artifacts/public/home.tar.xz" + - "tf_win-amd64-gpu-opt" + tensorflow: ${system.tensorflow.win_amd64_cuda.url} scripts: build: "taskcluster/win-build.sh --cuda" package: "taskcluster/win-package.sh" diff --git a/taskcluster/win-amd64-tflite-opt.yml b/taskcluster/win-amd64-tflite-opt.yml index 629380b7..7fc9fbbb 100644 --- a/taskcluster/win-amd64-tflite-opt.yml +++ b/taskcluster/win-amd64-tflite-opt.yml @@ -8,7 +8,8 @@ build: - "node-gyp-cache" - "swig-win-amd64" - "pyenv-win-amd64" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz" + - "tf_win-amd64-cpu-opt" + tensorflow: ${system.tensorflow.win_amd64_cpu.url} scripts: build: "taskcluster/win-build.sh --tflite" package: "taskcluster/win-package.sh" diff --git a/taskcluster/win-build.sh b/taskcluster/win-build.sh index 0a0b9210..f5951db9 100755 --- a/taskcluster/win-build.sh +++ b/taskcluster/win-build.sh @@ -6,7 +6,7 @@ package_option=$1 source $(dirname "$0")/tc-tests-utils.sh -source ${DS_ROOT_TASK}/DeepSpeech/tf/tc-vars.sh +source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so @@ -31,7 +31,7 @@ SYSTEM_TARGET=host-win do_bazel_build if [ "${package_option}" = "--cuda" ]; then - cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/tf/bazel-bin/native_client/libdeepspeech.so.if.lib + cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin/native_client/liblibdeepspeech.so.ifso ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel-bin/native_client/libdeepspeech.so.if.lib fi export PATH=$PATH:$(cygpath ${ChocolateyInstall})/bin:'/c/Program Files/nodejs/' diff --git a/taskcluster/win-opt-base.tyml b/taskcluster/win-opt-base.tyml index e7db49a4..dae7045c 100644 --- a/taskcluster/win-opt-base.tyml +++ b/taskcluster/win-opt-base.tyml @@ -40,8 +40,6 @@ payload: "C:\Program Files\7-zip\7z.exe" x -txz -so msys2-base-x86_64.tar.xz | "C:\Program Files\7-zip\7z.exe" x -o%USERPROFILE% -ttar -aoa -si - .\msys64\usr\bin\bash.exe --login -cx "export THIS_BASH_PID=$$; ps -ef | grep '[?]' | awk '{print $2}' | grep -v $THIS_BASH_PID | xargs -r kill; exit 0" - - .\msys64\usr\bin\bash.exe --login -cx "${system.msys2_filesystem_pkg.install}" - - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - echo .\msys64\usr\bin\bash.exe --login -cxe " export LC_ALL=C && @@ -54,9 +52,10 @@ payload: (7z x -txz -so $USERPROFILE/home.tar.xz | 7z x -aoa -ttar -si ) && git clone --quiet $EXAMPLES_CLONE_URL $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && git checkout --quiet $EXAMPLES_CHECKOUT_TARGET && - git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && - cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && - ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client && + cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} && + git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && + (rm $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tensorflow/native_client || true) && + ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tensorflow/native_client && cd $TASKCLUSTER_TASK_DIR && (mkdir pyenv-root/ && 7z x -so $USERPROFILE/pyenv.tar.gz | 7z x -opyenv-root/ -aoa -ttar -si ) && pacman --noconfirm -S tar make && @@ -75,10 +74,6 @@ payload: content: sha256: ${system.msys2.sha} url: ${system.msys2.url} - - file: filesystem-2020.02-3-x86_64.pkg.tar.xz - content: - sha256: ${system.msys2_filesystem_pkg.sha} - url: ${system.msys2_filesystem_pkg.url} - file: home.tar.xz content: url: ${build.tensorflow} diff --git a/taskcluster/win-package.sh b/taskcluster/win-package.sh index f916a91c..96c1d7fa 100755 --- a/taskcluster/win-package.sh +++ b/taskcluster/win-package.sh @@ -6,7 +6,7 @@ source $(dirname "$0")/tc-tests-utils.sh mkdir -p ${TASKCLUSTER_ARTIFACTS} || true -cp ${DS_ROOT_TASK}/DeepSpeech/tf/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ +cp ${DS_ROOT_TASK}/DeepSpeech/ds/tensorflow/bazel*.log ${TASKCLUSTER_ARTIFACTS}/ package_native_client "native_client.tar.xz" diff --git a/tensorflow b/tensorflow new file mode 160000 index 00000000..7ead5580 --- /dev/null +++ b/tensorflow @@ -0,0 +1 @@ +Subproject commit 7ead55807a2ded84c107720ebca61e6285e2c239 From 837902ff10b719578d2b65caf9e4d9e341ca890e Mon Sep 17 00:00:00 2001 From: lissyx <1645737+lissyx@users.noreply.github.com> Date: Mon, 29 Jun 2020 22:37:37 +0200 Subject: [PATCH 05/20] Hotfix TensorFlow repo reference --- taskcluster/.shared.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index dcae80b7..ce031b29 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -142,32 +142,32 @@ system: namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" tensorflow: linux_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cpu/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cpu" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.cpu/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.cpu" linux_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cuda/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.cuda" linux_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.arm/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.arm" linux_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.arm64" darwin_amd64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.osx/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.osx" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.osx/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.osx" android_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.android-arm64" android_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-armv7/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.android-armv7" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.android-armv7" win_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.win/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.win" win_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win-cuda/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.0.win-cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.win-cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.win-cuda" username: 'build-user' homedir: linux: '/home/build-user' From f365576517009ea2de6d74ff859d8e8bcdef7037 Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 30 Jun 2020 00:53:12 +0200 Subject: [PATCH 06/20] Set git remote origin before fetching --- taskcluster/darwin-opt-base.tyml | 2 +- taskcluster/linux-opt-base.tyml | 2 +- taskcluster/win-opt-base.tyml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index 1c5b2faf..ea7ba95d 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -60,7 +60,7 @@ payload: cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ && exit $TASKCLUSTER_TASK_EXIT_CODE" 0 && (pixz -d < $TASKCLUSTER_ORIG_TASKDIR/home.tar.xz | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) && - cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} && + cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git remote set-url origin ${event.head.repo.url} && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && cd $TASKCLUSTER_TASK_DIR && (mkdir pyenv-root/ && gtar -C pyenv-root/ -xf $TASKCLUSTER_ORIG_TASKDIR/pyenv.tar.gz) && diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index 88da6db1..188c04bf 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -46,7 +46,7 @@ then: adduser --system --home ${system.homedir.linux} ${system.username} && apt-get -qq update && apt-get -qq -y install ${deepspeech.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} && cd ${system.homedir.linux}/ && - echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && cd ~/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && + echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && cd ~/DeepSpeech/ds && git remote set-url origin ${event.head.repo.url} && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && sudo -H -u ${system.username} --preserve-env /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.build} && sudo -H -u ${system.username} /bin/bash ${system.homedir.linux}/DeepSpeech/ds/${build.scripts.package} diff --git a/taskcluster/win-opt-base.tyml b/taskcluster/win-opt-base.tyml index dae7045c..b402e4b2 100644 --- a/taskcluster/win-opt-base.tyml +++ b/taskcluster/win-opt-base.tyml @@ -52,7 +52,7 @@ payload: (7z x -txz -so $USERPROFILE/home.tar.xz | 7z x -aoa -ttar -si ) && git clone --quiet $EXAMPLES_CLONE_URL $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/examples && git checkout --quiet $EXAMPLES_CHECKOUT_TARGET && - cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git fetch origin && git checkout --quiet ${event.head.sha} && + cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git remote set-url origin ${event.head.repo.url} && git fetch origin && git checkout --quiet ${event.head.sha} && git submodule --quiet sync tensorflow/ && git submodule --quiet update tensorflow/ && (rm $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tensorflow/native_client || true) && ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/tensorflow/native_client && From 03ca94887c62cff6de755cd86d5b6c76c83fe1a0 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Fri, 26 Jun 2020 10:23:46 +0200 Subject: [PATCH 07/20] Move DS_ErrorCodeToErrorMessage impl to its own object so it can be used without including all of libdeepspeech --- native_client/BUILD | 7 ++++--- native_client/deepspeech.cc | 17 ----------------- native_client/deepspeech_errors.cc | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 native_client/deepspeech_errors.cc diff --git a/native_client/BUILD b/native_client/BUILD index 53711dc2..965a766c 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -89,13 +89,14 @@ cc_library( tf_cc_shared_object( name = "libdeepspeech.so", srcs = [ + "alphabet.h", "deepspeech.cc", "deepspeech.h", - "alphabet.h", - "modelstate.h", + "deepspeech_errors.cc", "modelstate.cc", - "workspace_status.h", + "modelstate.h", "workspace_status.cc", + "workspace_status.h", ] + select({ "//native_client:tflite": [ "tflitemodelstate.h", diff --git a/native_client/deepspeech.cc b/native_client/deepspeech.cc index 3bcecc60..38868d4b 100644 --- a/native_client/deepspeech.cc +++ b/native_client/deepspeech.cc @@ -501,20 +501,3 @@ DS_Version() { return strdup(ds_version()); } - -char* -DS_ErrorCodeToErrorMessage(int aErrorCode) -{ -#define RETURN_MESSAGE(NAME, VALUE, DESC) \ - case NAME: \ - return strdup(DESC); - - switch(aErrorCode) - { - DS_FOR_EACH_ERROR(RETURN_MESSAGE) - default: - return strdup("Unknown error, please make sure you are using the correct native binary."); - } - -#undef RETURN_MESSAGE -} diff --git a/native_client/deepspeech_errors.cc b/native_client/deepspeech_errors.cc new file mode 100644 index 00000000..1f1e4d8d --- /dev/null +++ b/native_client/deepspeech_errors.cc @@ -0,0 +1,19 @@ +#include "deepspeech.h" +#include + +char* +DS_ErrorCodeToErrorMessage(int aErrorCode) +{ +#define RETURN_MESSAGE(NAME, VALUE, DESC) \ + case NAME: \ + return strdup(DESC); + + switch(aErrorCode) + { + DS_FOR_EACH_ERROR(RETURN_MESSAGE) + default: + return strdup("Unknown error, please make sure you are using the correct native binary."); + } + +#undef RETURN_MESSAGE +} From f82c77392de3e6dbf8c28471adb7db3b6ab83937 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Fri, 26 Jun 2020 10:27:35 +0200 Subject: [PATCH 08/20] Rewrite data/lm/generate_package.py into native_client/generate_scorer_package.cpp --- data/lm/generate_package.py | 157 ---------------------- native_client/BUILD | 19 +++ native_client/alphabet.h | 45 ++++++- native_client/ctcdecode/scorer.cpp | 2 +- native_client/ctcdecode/scorer.h | 3 +- native_client/generate_scorer_package.cpp | 146 ++++++++++++++++++++ 6 files changed, 211 insertions(+), 161 deletions(-) delete mode 100644 data/lm/generate_package.py create mode 100644 native_client/generate_scorer_package.cpp diff --git a/data/lm/generate_package.py b/data/lm/generate_package.py deleted file mode 100644 index 30a33fcc..00000000 --- a/data/lm/generate_package.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python -from __future__ import absolute_import, division, print_function - -import argparse -import shutil -import sys - -import ds_ctcdecoder -from deepspeech_training.util.text import Alphabet, UTF8Alphabet -from ds_ctcdecoder import Scorer, Alphabet as NativeAlphabet - - -def create_bundle( - alphabet_path, - lm_path, - vocab_path, - package_path, - force_utf8, - default_alpha, - default_beta, -): - words = set() - vocab_looks_char_based = True - with open(vocab_path) as fin: - for line in fin: - for word in line.split(): - words.add(word.encode("utf-8")) - if len(word) > 1: - vocab_looks_char_based = False - print("{} unique words read from vocabulary file.".format(len(words))) - - cbm = "Looks" if vocab_looks_char_based else "Doesn't look" - print("{} like a character based model.".format(cbm)) - - if force_utf8 != None: # pylint: disable=singleton-comparison - use_utf8 = force_utf8.value - else: - use_utf8 = vocab_looks_char_based - print("Using detected UTF-8 mode: {}".format(use_utf8)) - - if use_utf8: - serialized_alphabet = UTF8Alphabet().serialize() - else: - if not alphabet_path: - raise RuntimeError("No --alphabet path specified, can't continue.") - serialized_alphabet = Alphabet(alphabet_path).serialize() - - alphabet = NativeAlphabet() - err = alphabet.deserialize(serialized_alphabet, len(serialized_alphabet)) - if err != 0: - raise RuntimeError("Error loading alphabet: {}".format(err)) - - scorer = Scorer() - scorer.set_alphabet(alphabet) - scorer.set_utf8_mode(use_utf8) - scorer.reset_params(default_alpha, default_beta) - err = scorer.load_lm(lm_path) - if err != ds_ctcdecoder.DS_ERR_SCORER_NO_TRIE: - print('Error loading language model file: 0x{:X}.'.format(err)) - print('See the error codes section in https://deepspeech.readthedocs.io for a description.') - sys.exit(1) - scorer.fill_dictionary(list(words)) - shutil.copy(lm_path, package_path) - # append, not overwrite - if scorer.save_dictionary(package_path, True): - print("Package created in {}".format(package_path)) - else: - print("Error when creating {}".format(package_path)) - sys.exit(1) - - -class Tristate(object): - def __init__(self, value=None): - if any(value is v for v in (True, False, None)): - self.value = value - else: - raise ValueError("Tristate value must be True, False, or None") - - def __eq__(self, other): - return ( - self.value is other.value - if isinstance(other, Tristate) - else self.value is other - ) - - def __ne__(self, other): - return not self == other - - def __bool__(self): - raise TypeError("Tristate object may not be used as a Boolean") - - def __str__(self): - return str(self.value) - - def __repr__(self): - return "Tristate(%s)" % self.value - - -def main(): - parser = argparse.ArgumentParser( - description="Generate an external scorer package for DeepSpeech." - ) - parser.add_argument( - "--alphabet", - help="Path of alphabet file to use for vocabulary construction. Words with characters not in the alphabet will not be included in the vocabulary. Optional if using UTF-8 mode.", - ) - parser.add_argument( - "--lm", - required=True, - help="Path of KenLM binary LM file. Must be built without including the vocabulary (use the -v flag). See generate_lm.py for how to create a binary LM.", - ) - parser.add_argument( - "--vocab", - required=True, - help="Path of vocabulary file. Must contain words separated by whitespace.", - ) - parser.add_argument("--package", required=True, help="Path to save scorer package.") - parser.add_argument( - "--default_alpha", - type=float, - required=True, - help="Default value of alpha hyperparameter.", - ) - parser.add_argument( - "--default_beta", - type=float, - required=True, - help="Default value of beta hyperparameter.", - ) - parser.add_argument( - "--force_utf8", - type=str, - default="", - help="Boolean flag, force set or unset UTF-8 mode in the scorer package. If not set, infers from the vocabulary. See for further explanation", - ) - args = parser.parse_args() - - if args.force_utf8 in ("True", "1", "true", "yes", "y"): - force_utf8 = Tristate(True) - elif args.force_utf8 in ("False", "0", "false", "no", "n"): - force_utf8 = Tristate(False) - else: - force_utf8 = Tristate(None) - - create_bundle( - args.alphabet, - args.lm, - args.vocab, - args.package, - force_utf8, - args.default_alpha, - args.default_beta, - ) - - -if __name__ == "__main__": - main() diff --git a/native_client/BUILD b/native_client/BUILD index 965a766c..36702088 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -2,6 +2,7 @@ load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_cc_shared_object") load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") +load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps") load( "@org_tensorflow//tensorflow/lite:build_def.bzl", @@ -78,6 +79,8 @@ cc_library( hdrs = [ "ctcdecode/ctc_beam_search_decoder.h", "ctcdecode/scorer.h", + "ctcdecode/decoder_utils.h", + "alphabet.h", ], includes = [ ".", @@ -186,6 +189,22 @@ genrule( cmd = "dsymutil $(location :libdeepspeech.so) -o $@" ) +cc_binary( + name = "generate_scorer_package", + srcs = [ + "generate_scorer_package.cpp", + "deepspeech_errors.cc", + ], + copts = ["-std=c++11"], + deps = [ + ":decoder", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/types:optional", + "@boost//:program_options", + ], +) + cc_binary( name = "enumerate_kenlm_vocabulary", srcs = [ diff --git a/native_client/alphabet.h b/native_client/alphabet.h index ace905cc..e57ef914 100644 --- a/native_client/alphabet.h +++ b/native_client/alphabet.h @@ -19,7 +19,7 @@ public: Alphabet(const Alphabet&) = default; Alphabet& operator=(const Alphabet&) = default; - int init(const char *config_file) { + virtual int init(const char *config_file) { std::ifstream in(config_file, std::ios::in); if (!in) { return 1; @@ -45,6 +45,30 @@ public: return 0; } + std::string serialize() { + // Serialization format is a sequence of (key, value) pairs, where key is + // a uint16_t and value is a uint16_t length followed by `length` UTF-8 + // encoded bytes with the label. + std::stringstream out; + + // We start by writing the number of pairs in the buffer as uint16_t. + uint16_t size = size_; + out.write(reinterpret_cast(&size), sizeof(size)); + + for (auto it = label_to_str_.begin(); it != label_to_str_.end(); ++it) { + uint16_t key = it->first; + string str = it->second; + uint16_t len = str.length(); + // Then we write the key as uint16_t, followed by the length of the value + // as uint16_t, followed by `length` bytes (the value itself). + out.write(reinterpret_cast(&key), sizeof(key)); + out.write(reinterpret_cast(&len), sizeof(len)); + out.write(str.data(), len); + } + + return out.str(); + } + int deserialize(const char* buffer, const int buffer_size) { // See util/text.py for an explanation of the serialization format. int offset = 0; @@ -126,11 +150,28 @@ public: return word; } -private: +protected: size_t size_; unsigned int space_label_; std::unordered_map label_to_str_; std::unordered_map str_to_label_; }; +class UTF8Alphabet : public Alphabet +{ +public: + UTF8Alphabet() { + size_ = 255; + space_label_ = ' ' - 1; + for (int i = 0; i < size_; ++i) { + std::string val(1, i+1); + label_to_str_[i] = val; + str_to_label_[val] = i; + } + } + + int init(const char*) override {} +}; + + #endif //ALPHABET_H diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp index ebf55227..401613d1 100644 --- a/native_client/ctcdecode/scorer.cpp +++ b/native_client/ctcdecode/scorer.cpp @@ -357,7 +357,7 @@ std::vector Scorer::make_ngram(PathTrie* prefix) return ngram; } -void Scorer::fill_dictionary(const std::vector& vocabulary) +void Scorer::fill_dictionary(const std::unordered_set& vocabulary) { // ConstFst is immutable, so we need to use a MutableFst to create the trie, // and then we convert to a ConstFst for the decoder and for storing on disk. diff --git a/native_client/ctcdecode/scorer.h b/native_client/ctcdecode/scorer.h index d2a1c8b3..3e7c0761 100644 --- a/native_client/ctcdecode/scorer.h +++ b/native_client/ctcdecode/scorer.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "lm/virtual_interface.hh" @@ -83,7 +84,7 @@ public: bool is_scoring_boundary(PathTrie* prefix, size_t new_label); // fill dictionary FST from a vocabulary - void fill_dictionary(const std::vector &vocabulary); + void fill_dictionary(const std::unordered_set &vocabulary); // load language model from given path int load_lm(const std::string &lm_path); diff --git a/native_client/generate_scorer_package.cpp b/native_client/generate_scorer_package.cpp new file mode 100644 index 00000000..910bf9c2 --- /dev/null +++ b/native_client/generate_scorer_package.cpp @@ -0,0 +1,146 @@ +#include +#include +#include +#include +#include +using namespace std; + +#include "absl/types/optional.h" +#include "boost/program_options.hpp" + +#include "ctcdecode/decoder_utils.h" +#include "ctcdecode/scorer.h" +#include "alphabet.h" +#include "deepspeech.h" + +namespace po = boost::program_options; + +int +create_package(absl::optional alphabet_path, + string lm_path, + string vocab_path, + string package_path, + absl::optional force_utf8, + float default_alpha, + float default_beta) +{ + // Read vocabulary + unordered_set words; + bool vocab_looks_char_based = true; + ifstream fin(vocab_path); + if (!fin) { + cerr << "Invalid vocabulary file " << vocab_path << "\n"; + return 1; + } + string word; + while (fin >> word) { + words.insert(word); + if (get_utf8_str_len(word) > 1) { + vocab_looks_char_based = false; + } + } + cerr << words.size() << " unique words read from vocabulary file.\n" + << (vocab_looks_char_based ? "Looks" : "Doesn't look") + << " like a character based (Bytes Are All You Need) model.\n"; + + if (!force_utf8.has_value()) { + force_utf8 = vocab_looks_char_based; + cerr << "--force_utf8 was not specified, using value " + << "infered from vocabulary contents: " + << (vocab_looks_char_based ? "true" : "false") << "\n"; + } + + if (force_utf8.value() && !alphabet_path.has_value()) { + cerr << "No --alphabet file specified, not using bytes output mode, can't continue.\n"; + return 1; + } + + Scorer scorer; + if (force_utf8.value()) { + scorer.set_alphabet(UTF8Alphabet()); + } else { + Alphabet alphabet; + alphabet.init(alphabet_path->c_str()); + scorer.set_alphabet(alphabet); + } + scorer.set_utf8_mode(force_utf8.value()); + scorer.reset_params(default_alpha, default_beta); + int err = scorer.load_lm(lm_path); + if (err != DS_ERR_SCORER_NO_TRIE) { + cerr << "Error loading language model file: " + << DS_ErrorCodeToErrorMessage(err) << "\n"; + return 1; + } + scorer.fill_dictionary(words); + + // Copy LM file to final package file destination + { + ifstream lm_src(lm_path, std::ios::binary); + ofstream package_dest(package_path, std::ios::binary); + package_dest << lm_src.rdbuf(); + } + + // Save dictionary to package file, appending instead of overwriting + if (!scorer.save_dictionary(package_path, true)) { + cerr << "Error when saving package in " << package_path << ".\n"; + return 1; + } + + cerr << "Package created in " << package_path << ".\n"; + return 0; +} + +int +main(int argc, char** argv) +{ + po::options_description desc("Options"); + desc.add_options() + ("help", "show help message") + ("alphabet", po::value(), "Path of alphabet file to use for vocabulary construction. Words with characters not in the alphabet will not be included in the vocabulary. Optional if using UTF-8 mode.") + ("lm", po::value(), "Path of KenLM binary LM file. Must be built without including the vocabulary (use the -v flag). See generate_lm.py for how to create a binary LM.") + ("vocab", po::value(), "Path of vocabulary file. Must contain words separated by whitespace.") + ("package", po::value(), "Path to save scorer package.") + ("default_alpha", po::value(), "Default value of alpha hyperparameter (float).") + ("default_beta", po::value(), "Default value of beta hyperparameter (float).") + ("force_utf8", po::value(), "Boolean flag, force set or unset UTF-8 mode in the scorer package. If not set, infers from the vocabulary. See for further explanation.") + ; + + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + po::notify(vm); + + if (vm.count("help")) { + cout << desc << "\n"; + return 1; + } + + // Check required flags. + for (const string& flag : {"lm", "vocab", "package", "default_alpha", "default_beta"}) { + if (!vm.count(flag)) { + cerr << "--" << flag << " is a required flag. Pass --help for help.\n"; + return 1; + } + } + + // Parse optional --force_utf8 + absl::optional force_utf8 = absl::nullopt; + if (vm.count("force_utf8")) { + force_utf8 = vm["force_utf8"].as(); + } + + // Parse optional --alphabet + absl::optional alphabet = absl::nullopt; + if (vm.count("alphabet")) { + alphabet = vm["alphabet"].as(); + } + + create_package(alphabet, + vm["lm"].as(), + vm["vocab"].as(), + vm["package"].as(), + force_utf8, + vm["default_alpha"].as(), + vm["default_beta"].as()); + + return 0; +} From a84abf813cd0cbc3257429aa44d00a3552f98f71 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Fri, 26 Jun 2020 11:57:10 +0200 Subject: [PATCH 09/20] Deduplicate Alphabet implementations, use C++ one everywhere --- native_client/BUILD | 19 ++- native_client/alphabet.cc | 154 +++++++++++++++++++ native_client/alphabet.h | 146 ++++-------------- native_client/ctcdecode/__init__.py | 42 ++--- native_client/ctcdecode/build_archive.py | 3 +- native_client/ctcdecode/decoder_utils.cpp | 4 +- native_client/ctcdecode/decoder_utils.h | 2 +- native_client/ctcdecode/output.h | 4 +- native_client/ctcdecode/path_trie.cpp | 18 +-- native_client/ctcdecode/path_trie.h | 18 +-- native_client/ctcdecode/scorer.cpp | 12 +- native_client/ctcdecode/scorer.h | 2 +- native_client/ctcdecode/swigwrapper.i | 9 +- native_client/modelstate.cc | 4 +- native_client/tflitemodelstate.cc | 2 +- native_client/tfmodelstate.cc | 2 +- tests/test_text.py | 6 +- training/deepspeech_training/evaluate.py | 2 +- training/deepspeech_training/train.py | 2 +- training/deepspeech_training/util/config.py | 5 +- training/deepspeech_training/util/helpers.py | 6 +- training/deepspeech_training/util/text.py | 117 +------------- 22 files changed, 257 insertions(+), 322 deletions(-) create mode 100644 native_client/alphabet.cc diff --git a/native_client/BUILD b/native_client/BUILD index 36702088..232d99c7 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -75,6 +75,7 @@ cc_library( "ctcdecode/scorer.cpp", "ctcdecode/path_trie.cpp", "ctcdecode/path_trie.h", + "alphabet.cc", ] + OPENFST_SOURCES_PLATFORM, hdrs = [ "ctcdecode/ctc_beam_search_decoder.h", @@ -86,13 +87,17 @@ cc_library( ".", "ctcdecode/third_party/ThreadPool", ] + OPENFST_INCLUDES_PLATFORM, - deps = [":kenlm"] + deps = [":kenlm"], + linkopts = [ + "-lm", + "-ldl", + "-pthread", + ], ) tf_cc_shared_object( name = "libdeepspeech.so", srcs = [ - "alphabet.h", "deepspeech.cc", "deepspeech.h", "deepspeech_errors.cc", @@ -203,6 +208,11 @@ cc_binary( "@com_google_absl//absl/types:optional", "@boost//:program_options", ], + linkopts = [ + "-lm", + "-ldl", + "-pthread", + ], ) cc_binary( @@ -221,10 +231,5 @@ cc_binary( "trie_load.cc", ], copts = ["-std=c++11"], - linkopts = [ - "-lm", - "-ldl", - "-pthread", - ], deps = [":decoder"], ) diff --git a/native_client/alphabet.cc b/native_client/alphabet.cc new file mode 100644 index 00000000..873b4881 --- /dev/null +++ b/native_client/alphabet.cc @@ -0,0 +1,154 @@ +#include "alphabet.h" +#include "ctcdecode/decoder_utils.h" + +#include + +int +Alphabet::init(const char *config_file) +{ + std::ifstream in(config_file, std::ios::in); + if (!in) { + return 1; + } + unsigned int label = 0; + space_label_ = -2; + for (std::string line; std::getline(in, line);) { + if (line.size() == 2 && line[0] == '\\' && line[1] == '#') { + line = '#'; + } else if (line[0] == '#') { + continue; + } + //TODO: we should probably do something more i18n-aware here + if (line == " ") { + space_label_ = label; + } + label_to_str_[label] = line; + str_to_label_[line] = label; + ++label; + } + size_ = label; + in.close(); + return 0; +} + +std::string +Alphabet::Serialize() +{ + // Serialization format is a sequence of (key, value) pairs, where key is + // a uint16_t and value is a uint16_t length followed by `length` UTF-8 + // encoded bytes with the label. + std::stringstream out; + + // We start by writing the number of pairs in the buffer as uint16_t. + uint16_t size = size_; + out.write(reinterpret_cast(&size), sizeof(size)); + + for (auto it = label_to_str_.begin(); it != label_to_str_.end(); ++it) { + uint16_t key = it->first; + string str = it->second; + uint16_t len = str.length(); + // Then we write the key as uint16_t, followed by the length of the value + // as uint16_t, followed by `length` bytes (the value itself). + out.write(reinterpret_cast(&key), sizeof(key)); + out.write(reinterpret_cast(&len), sizeof(len)); + out.write(str.data(), len); + } + + return out.str(); +} + +int +Alphabet::Deserialize(const char* buffer, const int buffer_size) +{ + // See util/text.py for an explanation of the serialization format. + int offset = 0; + if (buffer_size - offset < sizeof(uint16_t)) { + return 1; + } + uint16_t size = *(uint16_t*)(buffer + offset); + offset += sizeof(uint16_t); + size_ = size; + + for (int i = 0; i < size; ++i) { + if (buffer_size - offset < sizeof(uint16_t)) { + return 1; + } + uint16_t label = *(uint16_t*)(buffer + offset); + offset += sizeof(uint16_t); + + if (buffer_size - offset < sizeof(uint16_t)) { + return 1; + } + uint16_t val_len = *(uint16_t*)(buffer + offset); + offset += sizeof(uint16_t); + + if (buffer_size - offset < val_len) { + return 1; + } + std::string val(buffer+offset, val_len); + offset += val_len; + + label_to_str_[label] = val; + str_to_label_[val] = label; + + if (val == " ") { + space_label_ = label; + } + } + + return 0; +} + +std::string +Alphabet::DecodeSingle(unsigned int label) const +{ + auto it = label_to_str_.find(label); + if (it != label_to_str_.end()) { + return it->second; + } else { + std::cerr << "Invalid label " << label << std::endl; + abort(); + } +} + +unsigned int +Alphabet::EncodeSingle(const std::string& string) const +{ + auto it = str_to_label_.find(string); + if (it != str_to_label_.end()) { + return it->second; + } else { + std::cerr << "Invalid string " << string << std::endl; + abort(); + } +} + +std::string +Alphabet::Decode(const std::vector& input) const +{ + std::string word; + for (auto ind : input) { + word += DecodeSingle(ind); + } + return word; +} + +std::string +Alphabet::Decode(const unsigned int* input, int length) const +{ + std::string word; + for (int i = 0; i < length; ++i) { + word += DecodeSingle(input[i]); + } + return word; +} + +std::vector +Alphabet::Encode(const std::string& input) const +{ + std::vector result; + for (auto cp : split_into_codepoints(input)) { + result.push_back(EncodeSingle(cp)); + } + return result; +} diff --git a/native_client/alphabet.h b/native_client/alphabet.h index e57ef914..45fc444e 100644 --- a/native_client/alphabet.h +++ b/native_client/alphabet.h @@ -1,9 +1,6 @@ #ifndef ALPHABET_H #define ALPHABET_H -#include -#include -#include #include #include #include @@ -18,116 +15,15 @@ public: Alphabet() = default; Alphabet(const Alphabet&) = default; Alphabet& operator=(const Alphabet&) = default; + virtual ~Alphabet() = default; - virtual int init(const char *config_file) { - std::ifstream in(config_file, std::ios::in); - if (!in) { - return 1; - } - unsigned int label = 0; - space_label_ = -2; - for (std::string line; std::getline(in, line);) { - if (line.size() == 2 && line[0] == '\\' && line[1] == '#') { - line = '#'; - } else if (line[0] == '#') { - continue; - } - //TODO: we should probably do something more i18n-aware here - if (line == " ") { - space_label_ = label; - } - label_to_str_[label] = line; - str_to_label_[line] = label; - ++label; - } - size_ = label; - in.close(); - return 0; - } + virtual int init(const char *config_file); - std::string serialize() { - // Serialization format is a sequence of (key, value) pairs, where key is - // a uint16_t and value is a uint16_t length followed by `length` UTF-8 - // encoded bytes with the label. - std::stringstream out; + // Serialize alphabet into a binary buffer. + std::string Serialize(); - // We start by writing the number of pairs in the buffer as uint16_t. - uint16_t size = size_; - out.write(reinterpret_cast(&size), sizeof(size)); - - for (auto it = label_to_str_.begin(); it != label_to_str_.end(); ++it) { - uint16_t key = it->first; - string str = it->second; - uint16_t len = str.length(); - // Then we write the key as uint16_t, followed by the length of the value - // as uint16_t, followed by `length` bytes (the value itself). - out.write(reinterpret_cast(&key), sizeof(key)); - out.write(reinterpret_cast(&len), sizeof(len)); - out.write(str.data(), len); - } - - return out.str(); - } - - int deserialize(const char* buffer, const int buffer_size) { - // See util/text.py for an explanation of the serialization format. - int offset = 0; - if (buffer_size - offset < sizeof(uint16_t)) { - return 1; - } - uint16_t size = *(uint16_t*)(buffer + offset); - offset += sizeof(uint16_t); - size_ = size; - - for (int i = 0; i < size; ++i) { - if (buffer_size - offset < sizeof(uint16_t)) { - return 1; - } - uint16_t label = *(uint16_t*)(buffer + offset); - offset += sizeof(uint16_t); - - if (buffer_size - offset < sizeof(uint16_t)) { - return 1; - } - uint16_t val_len = *(uint16_t*)(buffer + offset); - offset += sizeof(uint16_t); - - if (buffer_size - offset < val_len) { - return 1; - } - std::string val(buffer+offset, val_len); - offset += val_len; - - label_to_str_[label] = val; - str_to_label_[val] = label; - - if (val == " ") { - space_label_ = label; - } - } - - return 0; - } - - const std::string& StringFromLabel(unsigned int label) const { - auto it = label_to_str_.find(label); - if (it != label_to_str_.end()) { - return it->second; - } else { - std::cerr << "Invalid label " << label << std::endl; - abort(); - } - } - - unsigned int LabelFromString(const std::string& string) const { - auto it = str_to_label_.find(string); - if (it != str_to_label_.end()) { - return it->second; - } else { - std::cerr << "Invalid string " << string << std::endl; - abort(); - } - } + // Deserialize alphabet from a binary buffer. + int Deserialize(const char* buffer, const int buffer_size); size_t GetSize() const { return size_; @@ -141,14 +37,22 @@ public: return space_label_; } - template - std::string LabelsToString(const std::vector& input) const { - std::string word; - for (auto ind : input) { - word += StringFromLabel(ind); - } - return word; - } + // Decode a single label into a string. + std::string DecodeSingle(unsigned int label) const; + + // Encode a single character/output class into a label. + unsigned int EncodeSingle(const std::string& string) const; + + // Decode a sequence of labels into a string. + std::string Decode(const std::vector& input) const; + + // We provide a C-style overload for accepting NumPy arrays as input, since + // the NumPy library does not have built-in typemaps for std::vector. + std::string Decode(const unsigned int* input, int length) const; + + // Encode a sequence of character/output classes into a sequence of labels. + // Characters are assumed to always take a single Unicode codepoint. + std::vector Encode(const std::string& input) const; protected: size_t size_; @@ -163,14 +67,16 @@ public: UTF8Alphabet() { size_ = 255; space_label_ = ' ' - 1; - for (int i = 0; i < size_; ++i) { + for (size_t i = 0; i < size_; ++i) { std::string val(1, i+1); label_to_str_[i] = val; str_to_label_[val] = i; } } - int init(const char*) override {} + int init(const char*) override { + return 0; + } }; diff --git a/native_client/ctcdecode/__init__.py b/native_client/ctcdecode/__init__.py index 7e3766be..ac603aa9 100644 --- a/native_client/ctcdecode/__init__.py +++ b/native_client/ctcdecode/__init__.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function from . import swigwrapper # pylint: disable=import-self -from .swigwrapper import Alphabet +from .swigwrapper import UTF8Alphabet __version__ = swigwrapper.__version__ @@ -30,24 +30,20 @@ class Scorer(swigwrapper.Scorer): assert beta is not None, 'beta parameter is required' assert scorer_path, 'scorer_path parameter is required' - serialized = alphabet.serialize() - native_alphabet = swigwrapper.Alphabet() - err = native_alphabet.deserialize(serialized, len(serialized)) + err = self.init(scorer_path, alphabet) if err != 0: - raise ValueError('Error when deserializing alphabet.') - - err = self.init(scorer_path.encode('utf-8'), - native_alphabet) - if err != 0: - raise ValueError('Scorer initialization failed with error code {}'.format(err)) + raise ValueError('Scorer initialization failed with error code 0x{:X}'.format(err)) self.reset_params(alpha, beta) - def load_lm(self, lm_path): - return super(Scorer, self).load_lm(lm_path.encode('utf-8')) - def save_dictionary(self, save_path, *args, **kwargs): - return super(Scorer, self).save_dictionary(save_path.encode('utf-8'), *args, **kwargs) +class Alphabet(swigwrapper.Alphabet): + """Convenience wrapper for Alphabet which calls init in the constructor""" + def __init__(self, config_path): + super(Alphabet, self).__init__() + err = self.init(config_path) + if err != 0: + raise ValueError('Alphabet initialization failed with error code 0x{:X}'.format(err)) def ctc_beam_search_decoder(probs_seq, @@ -79,15 +75,10 @@ def ctc_beam_search_decoder(probs_seq, results, in descending order of the confidence. :rtype: list """ - serialized = alphabet.serialize() - native_alphabet = swigwrapper.Alphabet() - err = native_alphabet.deserialize(serialized, len(serialized)) - if err != 0: - raise ValueError("Error when deserializing alphabet.") beam_results = swigwrapper.ctc_beam_search_decoder( - probs_seq, native_alphabet, beam_size, cutoff_prob, cutoff_top_n, + probs_seq, alphabet, beam_size, cutoff_prob, cutoff_top_n, scorer) - beam_results = [(res.confidence, alphabet.decode(res.tokens)) for res in beam_results] + beam_results = [(res.confidence, alphabet.Decode(res.tokens)) for res in beam_results] return beam_results @@ -126,14 +117,9 @@ def ctc_beam_search_decoder_batch(probs_seq, results, in descending order of the confidence. :rtype: list """ - serialized = alphabet.serialize() - native_alphabet = swigwrapper.Alphabet() - err = native_alphabet.deserialize(serialized, len(serialized)) - if err != 0: - raise ValueError("Error when deserializing alphabet.") - batch_beam_results = swigwrapper.ctc_beam_search_decoder_batch(probs_seq, seq_lengths, native_alphabet, beam_size, num_processes, cutoff_prob, cutoff_top_n, scorer) + batch_beam_results = swigwrapper.ctc_beam_search_decoder_batch(probs_seq, seq_lengths, alphabet, beam_size, num_processes, cutoff_prob, cutoff_top_n, scorer) batch_beam_results = [ - [(res.confidence, alphabet.decode(res.tokens)) for res in beam_results] + [(res.confidence, alphabet.Decode(res.tokens)) for res in beam_results] for beam_results in batch_beam_results ] return batch_beam_results diff --git a/native_client/ctcdecode/build_archive.py b/native_client/ctcdecode/build_archive.py index c379d6b3..8a689ac0 100644 --- a/native_client/ctcdecode/build_archive.py +++ b/native_client/ctcdecode/build_archive.py @@ -46,7 +46,8 @@ CTC_DECODER_FILES = [ 'scorer.cpp', 'path_trie.cpp', 'decoder_utils.cpp', - 'workspace_status.cc' + 'workspace_status.cc', + '../alphabet.cc', ] def build_archive(srcs=[], out_name='', build_dir='temp_build/temp_build', debug=False, num_parallel=1): diff --git a/native_client/ctcdecode/decoder_utils.cpp b/native_client/ctcdecode/decoder_utils.cpp index ed244c3a..bb3e1c77 100644 --- a/native_client/ctcdecode/decoder_utils.cpp +++ b/native_client/ctcdecode/decoder_utils.cpp @@ -119,7 +119,7 @@ bool prefix_compare_external(const PathTrie *x, const PathTrie *y, const std::un } } -void add_word_to_fst(const std::vector &word, +void add_word_to_fst(const std::vector &word, fst::StdVectorFst *dictionary) { if (dictionary->NumStates() == 0) { fst::StdVectorFst::StateId start = dictionary->AddState(); @@ -144,7 +144,7 @@ bool add_word_to_dictionary( fst::StdVectorFst *dictionary) { auto characters = utf8 ? split_into_bytes(word) : split_into_codepoints(word); - std::vector int_word; + std::vector int_word; for (auto &c : characters) { auto int_c = char_map.find(c); diff --git a/native_client/ctcdecode/decoder_utils.h b/native_client/ctcdecode/decoder_utils.h index 3ba1d7e6..c51ea046 100644 --- a/native_client/ctcdecode/decoder_utils.h +++ b/native_client/ctcdecode/decoder_utils.h @@ -86,7 +86,7 @@ std::vector split_into_codepoints(const std::string &str); std::vector split_into_bytes(const std::string &str); // Add a word in index to the dicionary of fst -void add_word_to_fst(const std::vector &word, +void add_word_to_fst(const std::vector &word, fst::StdVectorFst *dictionary); // Return whether a byte is a code point boundary (not a continuation byte). diff --git a/native_client/ctcdecode/output.h b/native_client/ctcdecode/output.h index 10eb4228..bdfc8ee9 100644 --- a/native_client/ctcdecode/output.h +++ b/native_client/ctcdecode/output.h @@ -8,8 +8,8 @@ */ struct Output { double confidence; - std::vector tokens; - std::vector timesteps; + std::vector tokens; + std::vector timesteps; }; #endif // OUTPUT_H_ diff --git a/native_client/ctcdecode/path_trie.cpp b/native_client/ctcdecode/path_trie.cpp index 0c0ee98c..7a04f693 100644 --- a/native_client/ctcdecode/path_trie.cpp +++ b/native_client/ctcdecode/path_trie.cpp @@ -35,7 +35,7 @@ PathTrie::~PathTrie() { } } -PathTrie* PathTrie::get_path_trie(int new_char, int new_timestep, float cur_log_prob_c, bool reset) { +PathTrie* PathTrie::get_path_trie(unsigned int new_char, unsigned int new_timestep, float cur_log_prob_c, bool reset) { auto child = children_.begin(); for (; child != children_.end(); ++child) { if (child->first == new_char) { @@ -102,7 +102,7 @@ PathTrie* PathTrie::get_path_trie(int new_char, int new_timestep, float cur_log_ } } -void PathTrie::get_path_vec(std::vector& output, std::vector& timesteps) { +void PathTrie::get_path_vec(std::vector& output, std::vector& timesteps) { // Recursive call: recurse back until stop condition, then append data in // correct order as we walk back down the stack in the lines below. if (parent != nullptr) { @@ -114,8 +114,8 @@ void PathTrie::get_path_vec(std::vector& output, std::vector& timestep } } -PathTrie* PathTrie::get_prev_grapheme(std::vector& output, - std::vector& timesteps, +PathTrie* PathTrie::get_prev_grapheme(std::vector& output, + std::vector& timesteps, const Alphabet& alphabet) { PathTrie* stop = this; @@ -124,7 +124,7 @@ PathTrie* PathTrie::get_prev_grapheme(std::vector& output, } // Recursive call: recurse back until stop condition, then append data in // correct order as we walk back down the stack in the lines below. - if (!byte_is_codepoint_boundary(alphabet.StringFromLabel(character)[0])) { + if (!byte_is_codepoint_boundary(alphabet.DecodeSingle(character)[0])) { stop = parent->get_prev_grapheme(output, timesteps, alphabet); } output.push_back(character); @@ -135,7 +135,7 @@ PathTrie* PathTrie::get_prev_grapheme(std::vector& output, int PathTrie::distance_to_codepoint_boundary(unsigned char *first_byte, const Alphabet& alphabet) { - if (byte_is_codepoint_boundary(alphabet.StringFromLabel(character)[0])) { + if (byte_is_codepoint_boundary(alphabet.DecodeSingle(character)[0])) { *first_byte = (unsigned char)character + 1; return 1; } @@ -146,8 +146,8 @@ int PathTrie::distance_to_codepoint_boundary(unsigned char *first_byte, return 0; } -PathTrie* PathTrie::get_prev_word(std::vector& output, - std::vector& timesteps, +PathTrie* PathTrie::get_prev_word(std::vector& output, + std::vector& timesteps, const Alphabet& alphabet) { PathTrie* stop = this; @@ -225,7 +225,7 @@ void PathTrie::print(const Alphabet& a) { for (PathTrie* el : chain) { printf("%X ", (unsigned char)(el->character)); if (el->character != ROOT_) { - tr.append(a.StringFromLabel(el->character)); + tr.append(a.DecodeSingle(el->character)); } } printf("\ntimesteps:\t "); diff --git a/native_client/ctcdecode/path_trie.h b/native_client/ctcdecode/path_trie.h index dbd8a233..0a4374fc 100644 --- a/native_client/ctcdecode/path_trie.h +++ b/native_client/ctcdecode/path_trie.h @@ -21,22 +21,22 @@ public: ~PathTrie(); // get new prefix after appending new char - PathTrie* get_path_trie(int new_char, int new_timestep, float log_prob_c, bool reset = true); + PathTrie* get_path_trie(unsigned int new_char, unsigned int new_timestep, float log_prob_c, bool reset = true); // get the prefix data in correct time order from root to current node - void get_path_vec(std::vector& output, std::vector& timesteps); + void get_path_vec(std::vector& output, std::vector& timesteps); // get the prefix data in correct time order from beginning of last grapheme to current node - PathTrie* get_prev_grapheme(std::vector& output, - std::vector& timesteps, + PathTrie* get_prev_grapheme(std::vector& output, + std::vector& timesteps, const Alphabet& alphabet); // get the distance from current node to the first codepoint boundary, and the byte value at the boundary int distance_to_codepoint_boundary(unsigned char *first_byte, const Alphabet& alphabet); // get the prefix data in correct time order from beginning of last word to current node - PathTrie* get_prev_word(std::vector& output, - std::vector& timesteps, + PathTrie* get_prev_word(std::vector& output, + std::vector& timesteps, const Alphabet& alphabet); // update log probs @@ -64,8 +64,8 @@ public: float log_prob_c; float score; float approx_ctc; - int character; - int timestep; + unsigned int character; + unsigned int timestep; PathTrie* parent; private: @@ -73,7 +73,7 @@ private: bool exists_; bool has_dictionary_; - std::vector> children_; + std::vector> children_; // pointer to dictionary of FST std::shared_ptr dictionary_; diff --git a/native_client/ctcdecode/scorer.cpp b/native_client/ctcdecode/scorer.cpp index 401613d1..a6616e21 100644 --- a/native_client/ctcdecode/scorer.cpp +++ b/native_client/ctcdecode/scorer.cpp @@ -65,7 +65,7 @@ void Scorer::setup_char_map() // The initial state of FST is state 0, hence the index of chars in // the FST should start from 1 to avoid the conflict with the initial // state, otherwise wrong decoding results would be given. - char_map_[alphabet_.StringFromLabel(i)] = i + 1; + char_map_[alphabet_.DecodeSingle(i)] = i + 1; } } @@ -314,11 +314,11 @@ void Scorer::reset_params(float alpha, float beta) this->beta = beta; } -std::vector Scorer::split_labels_into_scored_units(const std::vector& labels) +std::vector Scorer::split_labels_into_scored_units(const std::vector& labels) { if (labels.empty()) return {}; - std::string s = alphabet_.LabelsToString(labels); + std::string s = alphabet_.Decode(labels); std::vector words; if (is_utf8_mode_) { words = split_into_codepoints(s); @@ -339,8 +339,8 @@ std::vector Scorer::make_ngram(PathTrie* prefix) break; } - std::vector prefix_vec; - std::vector prefix_steps; + std::vector prefix_vec; + std::vector prefix_steps; if (is_utf8_mode_) { new_node = current_node->get_prev_grapheme(prefix_vec, prefix_steps, alphabet_); @@ -350,7 +350,7 @@ std::vector Scorer::make_ngram(PathTrie* prefix) current_node = new_node->parent; // reconstruct word - std::string word = alphabet_.LabelsToString(prefix_vec); + std::string word = alphabet_.Decode(prefix_vec); ngram.push_back(word); } std::reverse(ngram.begin(), ngram.end()); diff --git a/native_client/ctcdecode/scorer.h b/native_client/ctcdecode/scorer.h index 3e7c0761..13c2ef1f 100644 --- a/native_client/ctcdecode/scorer.h +++ b/native_client/ctcdecode/scorer.h @@ -73,7 +73,7 @@ public: // trransform the labels in index to the vector of words (word based lm) or // the vector of characters (character based lm) - std::vector split_labels_into_scored_units(const std::vector &labels); + std::vector split_labels_into_scored_units(const std::vector &labels); void set_alphabet(const Alphabet& alphabet); diff --git a/native_client/ctcdecode/swigwrapper.i b/native_client/ctcdecode/swigwrapper.i index ab5675be..ffe23c3a 100644 --- a/native_client/ctcdecode/swigwrapper.i +++ b/native_client/ctcdecode/swigwrapper.i @@ -3,7 +3,6 @@ %{ #include "ctc_beam_search_decoder.h" #define SWIG_FILE_WITH_INIT -#define SWIG_PYTHON_STRICT_BYTE_CHAR #include "workspace_status.h" %} @@ -19,6 +18,9 @@ import_array(); namespace std { %template(StringVector) vector; + %template(UnsignedIntVector) vector; + %template(OutputVector) vector; + %template(OutputVectorVector) vector>; } %shared_ptr(Scorer); @@ -27,6 +29,7 @@ namespace std { %apply (double* IN_ARRAY2, int DIM1, int DIM2) {(const double *probs, int time_dim, int class_dim)}; %apply (double* IN_ARRAY3, int DIM1, int DIM2, int DIM3) {(const double *probs, int batch_size, int time_dim, int class_dim)}; %apply (int* IN_ARRAY1, int DIM1) {(const int *seq_lengths, int seq_lengths_size)}; +%apply (unsigned int* IN_ARRAY1, int DIM1) {(const unsigned int *input, int length)}; %ignore Scorer::dictionary; @@ -38,10 +41,6 @@ namespace std { %constant const char* __version__ = ds_version(); %constant const char* __git_version__ = ds_git_version(); -%template(IntVector) std::vector; -%template(OutputVector) std::vector; -%template(OutputVectorVector) std::vector>; - // Import only the error code enum definitions from deepspeech.h // We can't just do |%ignore "";| here because it affects this file globally (even // files %include'd above). That causes SWIG to lose destructor information and diff --git a/native_client/modelstate.cc b/native_client/modelstate.cc index 3cb06ac2..d8637c36 100644 --- a/native_client/modelstate.cc +++ b/native_client/modelstate.cc @@ -33,7 +33,7 @@ char* ModelState::decode(const DecoderState& state) const { vector out = state.decode(); - return strdup(alphabet_.LabelsToString(out[0].tokens).c_str()); + return strdup(alphabet_.Decode(out[0].tokens).c_str()); } Metadata* @@ -50,7 +50,7 @@ ModelState::decode_metadata(const DecoderState& state, for (int j = 0; j < out[i].tokens.size(); ++j) { TokenMetadata token { - strdup(alphabet_.StringFromLabel(out[i].tokens[j]).c_str()), // text + strdup(alphabet_.DecodeSingle(out[i].tokens[j]).c_str()), // text static_cast(out[i].timesteps[j]), // timestep out[i].timesteps[j] * ((float)audio_win_step_ / sample_rate_), // start_time }; diff --git a/native_client/tflitemodelstate.cc b/native_client/tflitemodelstate.cc index 4836ed0b..12be0d37 100644 --- a/native_client/tflitemodelstate.cc +++ b/native_client/tflitemodelstate.cc @@ -206,7 +206,7 @@ TFLiteModelState::init(const char* model_path) beam_width_ = (unsigned int)(*beam_width); tflite::StringRef serialized_alphabet = tflite::GetString(interpreter_->tensor(metadata_alphabet_idx), 0); - err = alphabet_.deserialize(serialized_alphabet.str, serialized_alphabet.len); + err = alphabet_.Deserialize(serialized_alphabet.str, serialized_alphabet.len); if (err != 0) { return DS_ERR_INVALID_ALPHABET; } diff --git a/native_client/tfmodelstate.cc b/native_client/tfmodelstate.cc index 440c44e6..65328e30 100644 --- a/native_client/tfmodelstate.cc +++ b/native_client/tfmodelstate.cc @@ -119,7 +119,7 @@ TFModelState::init(const char* model_path) beam_width_ = (unsigned int)(beam_width); string serialized_alphabet = metadata_outputs[4].scalar()(); - err = alphabet_.deserialize(serialized_alphabet.data(), serialized_alphabet.size()); + err = alphabet_.Deserialize(serialized_alphabet.data(), serialized_alphabet.size()); if (err != 0) { return DS_ERR_INVALID_ALPHABET; } diff --git a/tests/test_text.py b/tests/test_text.py index b26fda94..5bdda19e 100644 --- a/tests/test_text.py +++ b/tests/test_text.py @@ -1,7 +1,7 @@ import unittest import os -from deepspeech_training.util.text import Alphabet +from ds_ctcdecoder import Alphabet class TestAlphabetParsing(unittest.TestCase): @@ -11,12 +11,12 @@ class TestAlphabetParsing(unittest.TestCase): label_id = -1 for expected_label, expected_label_id in expected: try: - label_id = alphabet.encode(expected_label) + label_id = alphabet.Encode(expected_label) except KeyError: pass self.assertEqual(label_id, [expected_label_id]) try: - label = alphabet.decode([expected_label_id]) + label = alphabet.Decode([expected_label_id]) except KeyError: pass self.assertEqual(label, expected_label) diff --git a/training/deepspeech_training/evaluate.py b/training/deepspeech_training/evaluate.py index 716b5f93..00eac8c7 100755 --- a/training/deepspeech_training/evaluate.py +++ b/training/deepspeech_training/evaluate.py @@ -40,7 +40,7 @@ def sparse_tuple_to_texts(sp_tuple, alphabet): for i, index in enumerate(indices): results[index[0]].append(values[i]) # List of strings - return [alphabet.decode(res) for res in results] + return [alphabet.Decode(res) for res in results] def evaluate(test_csvs, create_model): diff --git a/training/deepspeech_training/train.py b/training/deepspeech_training/train.py index 175032da..93d0c727 100644 --- a/training/deepspeech_training/train.py +++ b/training/deepspeech_training/train.py @@ -771,7 +771,7 @@ def export(): outputs['metadata_feature_win_len'] = tf.constant([FLAGS.feature_win_len], name='metadata_feature_win_len') outputs['metadata_feature_win_step'] = tf.constant([FLAGS.feature_win_step], name='metadata_feature_win_step') outputs['metadata_beam_width'] = tf.constant([FLAGS.export_beam_width], name='metadata_beam_width') - outputs['metadata_alphabet'] = tf.constant([Config.alphabet.serialize()], name='metadata_alphabet') + outputs['metadata_alphabet'] = tf.constant([Config.alphabet.Serialize()], name='metadata_alphabet') if FLAGS.export_language: outputs['metadata_language'] = tf.constant([FLAGS.export_language.encode('utf-8')], name='metadata_language') diff --git a/training/deepspeech_training/util/config.py b/training/deepspeech_training/util/config.py index 13a36220..2bd580b5 100755 --- a/training/deepspeech_training/util/config.py +++ b/training/deepspeech_training/util/config.py @@ -6,14 +6,15 @@ import tensorflow.compat.v1 as tfv1 from attrdict import AttrDict from xdg import BaseDirectory as xdg +from ds_ctcdecoder import Alphabet, UTF8Alphabet from .flags import FLAGS from .gpu import get_available_gpus from .logging import log_error, log_warn -from .text import Alphabet, UTF8Alphabet from .helpers import parse_file_size from .augmentations import parse_augmentations + class ConfigSingleton: _config = None @@ -115,7 +116,7 @@ def initialize_globals(): c.n_hidden_3 = c.n_cell_dim # Units in the sixth layer = number of characters in the target language plus one - c.n_hidden_6 = c.alphabet.size() + 1 # +1 for CTC blank label + c.n_hidden_6 = c.alphabet.GetSize() + 1 # +1 for CTC blank label # Size of audio window in samples if (FLAGS.feature_win_len * FLAGS.audio_sample_rate) % 1000 != 0: diff --git a/training/deepspeech_training/util/helpers.py b/training/deepspeech_training/util/helpers.py index 6da708b9..32116f3f 100644 --- a/training/deepspeech_training/util/helpers.py +++ b/training/deepspeech_training/util/helpers.py @@ -52,12 +52,10 @@ def check_ctcdecoder_version(): sys.exit(1) raise e - decoder_version_s = decoder_version.decode() - - rv = semver.compare(ds_version_s, decoder_version_s) + rv = semver.compare(ds_version_s, decoder_version) if rv != 0: print("DeepSpeech version ({}) and CTC decoder version ({}) do not match. " - "Please ensure matching versions are in use.".format(ds_version_s, decoder_version_s)) + "Please ensure matching versions are in use.".format(ds_version_s, decoder_version)) sys.exit(1) return rv diff --git a/training/deepspeech_training/util/text.py b/training/deepspeech_training/util/text.py index 60bfe9f1..e1c2e981 100644 --- a/training/deepspeech_training/util/text.py +++ b/training/deepspeech_training/util/text.py @@ -3,121 +3,6 @@ from __future__ import absolute_import, division, print_function import numpy as np import struct -from six.moves import range - -class Alphabet(object): - def __init__(self, config_file): - self._config_file = config_file - self._label_to_str = {} - self._str_to_label = {} - self._size = 0 - if config_file: - with open(config_file, 'r', encoding='utf-8') as fin: - for line in fin: - if line[0:2] == '\\#': - line = '#\n' - elif line[0] == '#': - continue - self._label_to_str[self._size] = line[:-1] # remove the line ending - self._str_to_label[line[:-1]] = self._size - self._size += 1 - - def _string_from_label(self, label): - return self._label_to_str[label] - - def _label_from_string(self, string): - try: - return self._str_to_label[string] - except KeyError as e: - raise KeyError( - 'ERROR: Your transcripts contain characters (e.g. \'{}\') which do not occur in \'{}\'! Use ' \ - 'util/check_characters.py to see what characters are in your [train,dev,test].csv transcripts, and ' \ - 'then add all these to \'{}\'.'.format(string, self._config_file, self._config_file) - ).with_traceback(e.__traceback__) - - def has_char(self, char): - return char in self._str_to_label - - def encode(self, string): - res = [] - for char in string: - res.append(self._label_from_string(char)) - return res - - def decode(self, labels): - res = '' - for label in labels: - res += self._string_from_label(label) - return res - - def serialize(self): - # Serialization format is a sequence of (key, value) pairs, where key is - # a uint16_t and value is a uint16_t length followed by `length` UTF-8 - # encoded bytes with the label. - res = bytearray() - - # We start by writing the number of pairs in the buffer as uint16_t. - res += struct.pack(' Date: Sun, 28 Jun 2020 16:52:08 +0200 Subject: [PATCH 10/20] Build/package/publish generate_scorer_package in CI --- native_client/Android.mk | 2 +- taskcluster/android-build.sh | 1 + taskcluster/arm64-build.sh | 1 + taskcluster/cuda-build.sh | 1 + taskcluster/host-build.sh | 1 + taskcluster/rpi3-build.sh | 1 + taskcluster/tc-package.sh | 2 ++ taskcluster/win-build.sh | 1 + 8 files changed, 9 insertions(+), 1 deletion(-) diff --git a/native_client/Android.mk b/native_client/Android.mk index d21551fd..ee91a7a4 100644 --- a/native_client/Android.mk +++ b/native_client/Android.mk @@ -2,7 +2,7 @@ LOCAL_PATH := $(call my-dir) include $(CLEAR_VARS) LOCAL_MODULE := deepspeech-prebuilt -LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libdeepspeech.so +LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libdeepspeech.so $(TFDIR)/bazel-bin/native_client/generate_scorer_package include $(PREBUILT_SHARED_LIBRARY) include $(CLEAR_VARS) diff --git a/taskcluster/android-build.sh b/taskcluster/android-build.sh index 0121e51d..83d8eecb 100644 --- a/taskcluster/android-build.sh +++ b/taskcluster/android-build.sh @@ -10,6 +10,7 @@ source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so +//native_client:generate_scorer_package " if [ "${arm_flavor}" = "armeabi-v7a" ]; then diff --git a/taskcluster/arm64-build.sh b/taskcluster/arm64-build.sh index 7cfb7abf..1ca4028e 100644 --- a/taskcluster/arm64-build.sh +++ b/taskcluster/arm64-build.sh @@ -8,6 +8,7 @@ source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so +//native_client:generate_scorer_package " BAZEL_BUILD_FLAGS="${BAZEL_ARM64_FLAGS} ${BAZEL_EXTRA_FLAGS}" diff --git a/taskcluster/cuda-build.sh b/taskcluster/cuda-build.sh index dfaa236a..f8213f81 100755 --- a/taskcluster/cuda-build.sh +++ b/taskcluster/cuda-build.sh @@ -8,6 +8,7 @@ source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so +//native_client:generate_scorer_package " BAZEL_ENV_FLAGS="TF_NEED_CUDA=1 ${TF_CUDA_FLAGS}" diff --git a/taskcluster/host-build.sh b/taskcluster/host-build.sh index ddbc90fc..9ff3648c 100755 --- a/taskcluster/host-build.sh +++ b/taskcluster/host-build.sh @@ -10,6 +10,7 @@ source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so +//native_client:generate_scorer_package " if [ "${runtime}" = "tflite" ]; then diff --git a/taskcluster/rpi3-build.sh b/taskcluster/rpi3-build.sh index e9b795a4..eabff730 100755 --- a/taskcluster/rpi3-build.sh +++ b/taskcluster/rpi3-build.sh @@ -8,6 +8,7 @@ source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so +//native_client:generate_scorer_package " BAZEL_BUILD_FLAGS="${BAZEL_ARM_FLAGS} ${BAZEL_EXTRA_FLAGS}" diff --git a/taskcluster/tc-package.sh b/taskcluster/tc-package.sh index 03997991..ec36bb84 100755 --- a/taskcluster/tc-package.sh +++ b/taskcluster/tc-package.sh @@ -24,6 +24,7 @@ package_native_client() ${TAR} -cf - \ -C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so \ -C ${tensorflow_dir}/bazel-bin/native_client/ libdeepspeech.so.if.lib \ + -C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \ -C ${deepspeech_dir}/ LICENSE \ -C ${deepspeech_dir}/native_client/ deepspeech${PLATFORM_EXE_SUFFIX} \ -C ${deepspeech_dir}/native_client/ deepspeech.h \ @@ -56,6 +57,7 @@ package_native_client_ndk() tar -cf - \ -C ${deepspeech_dir}/native_client/libs/${arch_abi}/ deepspeech \ -C ${deepspeech_dir}/native_client/libs/${arch_abi}/ libdeepspeech.so \ + -C ${deepspeech_dir}/native_client/libs/${arch_abi}/ generate_scorer_package \ -C ${deepspeech_dir}/native_client/libs/${arch_abi}/ libc++_shared.so \ -C ${deepspeech_dir}/native_client/ deepspeech.h \ -C ${deepspeech_dir}/ LICENSE \ diff --git a/taskcluster/win-build.sh b/taskcluster/win-build.sh index f5951db9..48c7a19f 100755 --- a/taskcluster/win-build.sh +++ b/taskcluster/win-build.sh @@ -10,6 +10,7 @@ source $(dirname "$0")/tf_tc-vars.sh BAZEL_TARGETS=" //native_client:libdeepspeech.so +//native_client:generate_scorer_package " if [ "${package_option}" = "--cuda" ]; then From 6618148e9bf0ec003ef4223cd136e6e205ac292f Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Sun, 28 Jun 2020 17:00:27 +0200 Subject: [PATCH 11/20] Update tensorflow with Boost rules --- taskcluster/.build.yml | 2 +- taskcluster/.shared.yml | 36 ++++++++++++++++++------------------ tensorflow | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index 7fda06e6..fd261359 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -25,7 +25,7 @@ build: nc_asset_name: 'native_client.tar.xz' args: tests_cmdline: '' - tensorflow_git_desc: 'TensorFlow: v2.2.0-14-g7ead558' + tensorflow_git_desc: 'TensorFlow: v2.2.0-15-g518c1d0' test_model_task: '' homebrew: url: '' diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index ce031b29..46cd6983 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -142,32 +142,32 @@ system: namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" tensorflow: linux_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.cpu/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.cpu" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.cpu/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.cpu" linux_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.cuda/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.cuda" linux_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.arm/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.arm" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.arm/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.arm" linux_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.arm64" darwin_amd64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.osx/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.osx" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.osx/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.osx" android_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.android-arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.android-arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.android-arm64" android_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.android-armv7/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.android-armv7" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.android-armv7" win_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.win/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.win" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.win/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.win" win_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.win-cuda/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.7ead55807a2ded84c107720ebca61e6285e2c239.1.win-cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.win-cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.win-cuda" username: 'build-user' homedir: linux: '/home/build-user' diff --git a/tensorflow b/tensorflow index 7ead5580..518c1d04 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 7ead55807a2ded84c107720ebca61e6285e2c239 +Subproject commit 518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d From 5039fb51d5437970df0648b228404506fed359bc Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Mon, 29 Jun 2020 21:00:54 +0200 Subject: [PATCH 12/20] Package generate_scorer_package on Android --- native_client/Android.mk | 2 +- taskcluster/tc-package.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/native_client/Android.mk b/native_client/Android.mk index ee91a7a4..d21551fd 100644 --- a/native_client/Android.mk +++ b/native_client/Android.mk @@ -2,7 +2,7 @@ LOCAL_PATH := $(call my-dir) include $(CLEAR_VARS) LOCAL_MODULE := deepspeech-prebuilt -LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libdeepspeech.so $(TFDIR)/bazel-bin/native_client/generate_scorer_package +LOCAL_SRC_FILES := $(TFDIR)/bazel-bin/native_client/libdeepspeech.so include $(PREBUILT_SHARED_LIBRARY) include $(CLEAR_VARS) diff --git a/taskcluster/tc-package.sh b/taskcluster/tc-package.sh index ec36bb84..65280551 100755 --- a/taskcluster/tc-package.sh +++ b/taskcluster/tc-package.sh @@ -35,6 +35,7 @@ package_native_client() package_native_client_ndk() { deepspeech_dir=${DS_DSDIR} + tensorflow_dir=${DS_TFDIR} artifacts_dir=${TASKCLUSTER_ARTIFACTS} artifact_name=$1 arch_abi=$2 @@ -57,7 +58,7 @@ package_native_client_ndk() tar -cf - \ -C ${deepspeech_dir}/native_client/libs/${arch_abi}/ deepspeech \ -C ${deepspeech_dir}/native_client/libs/${arch_abi}/ libdeepspeech.so \ - -C ${deepspeech_dir}/native_client/libs/${arch_abi}/ generate_scorer_package \ + -C ${tensorflow_dir}/bazel-bin/native_client/ generate_scorer_package \ -C ${deepspeech_dir}/native_client/libs/${arch_abi}/ libc++_shared.so \ -C ${deepspeech_dir}/native_client/ deepspeech.h \ -C ${deepspeech_dir}/ LICENSE \ From 2504360e95a24c7c475edf8bb84b49dadcb8b978 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Mon, 29 Jun 2020 21:32:29 +0200 Subject: [PATCH 13/20] Handle universal newlines in Alphabet file parsing --- native_client/alphabet.cc | 37 ++++++++++++++++++++++++++++- native_client/ctcdecode/__init__.py | 5 ++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/native_client/alphabet.cc b/native_client/alphabet.cc index 873b4881..1f0a8dbe 100644 --- a/native_client/alphabet.cc +++ b/native_client/alphabet.cc @@ -3,6 +3,41 @@ #include +// std::getline, but handle newline conventions from multiple platforms instead +// of just the platform this code was built for +std::istream& +getline_crossplatform(std::istream& is, std::string& t) +{ + t.clear(); + + // The characters in the stream are read one-by-one using a std::streambuf. + // That is faster than reading them one-by-one using the std::istream. + // Code that uses streambuf this way must be guarded by a sentry object. + // The sentry object performs various tasks, + // such as thread synchronization and updating the stream state. + std::istream::sentry se(is, true); + std::streambuf* sb = is.rdbuf(); + + while (true) { + int c = sb->sbumpc(); + switch (c) { + case '\n': + return is; + case '\r': + if(sb->sgetc() == '\n') + sb->sbumpc(); + return is; + case std::streambuf::traits_type::eof(): + // Also handle the case when the last line has no line ending + if(t.empty()) + is.setstate(std::ios::eofbit); + return is; + default: + t += (char)c; + } + } +} + int Alphabet::init(const char *config_file) { @@ -12,7 +47,7 @@ Alphabet::init(const char *config_file) } unsigned int label = 0; space_label_ = -2; - for (std::string line; std::getline(in, line);) { + for (std::string line; getline_crossplatform(in, line);) { if (line.size() == 2 && line[0] == '\\' && line[1] == '#') { line = '#'; } else if (line[0] == '#') { diff --git a/native_client/ctcdecode/__init__.py b/native_client/ctcdecode/__init__.py index ac603aa9..ee5645d4 100644 --- a/native_client/ctcdecode/__init__.py +++ b/native_client/ctcdecode/__init__.py @@ -45,6 +45,11 @@ class Alphabet(swigwrapper.Alphabet): if err != 0: raise ValueError('Alphabet initialization failed with error code 0x{:X}'.format(err)) + def Encode(self, input): + """Convert SWIG's UnsignedIntVec to a Python list""" + res = super(Alphabet, self).Encode(input) + return [el for el in res] + def ctc_beam_search_decoder(probs_seq, alphabet, From a48ebdfde8a561161cec0f124099ca0f337ef8b5 Mon Sep 17 00:00:00 2001 From: Tilman Kamp <5991088+tilmankamp@users.noreply.github.com> Date: Tue, 30 Jun 2020 11:34:25 +0200 Subject: [PATCH 14/20] Reverb augmentation: Workaround for import problem in scikit-learn dependency of librosa --- training/deepspeech_training/util/augmentations.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/training/deepspeech_training/util/augmentations.py b/training/deepspeech_training/util/augmentations.py index eef58b81..7ac52c41 100644 --- a/training/deepspeech_training/util/augmentations.py +++ b/training/deepspeech_training/util/augmentations.py @@ -341,6 +341,8 @@ class Resample(SampleAugmentation): def apply(self, sample, clock=0.0): # late binding librosa and its dependencies + # pre-importing sklearn fixes https://github.com/scikit-learn/scikit-learn/issues/14485 + import sklearn # pylint: disable=import-outside-toplevel from librosa.core import resample # pylint: disable=import-outside-toplevel sample.change_audio_type(new_audio_type=AUDIO_TYPE_NP) rate = pick_value_from_range(self.rate, clock=clock) From 8f6106b35d2b84af160227640390590a2dc4c3ef Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 30 Jun 2020 16:47:41 +0200 Subject: [PATCH 15/20] Update docs to refer to new generate_scorer_package --- data/README.rst | 4 ++-- doc/Decoder.rst | 6 ++++-- doc/Scorer.rst | 16 ++++++++++------ training/deepspeech_training/util/flags.py | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/data/README.rst b/data/README.rst index 4d2c022a..aeb34864 100644 --- a/data/README.rst +++ b/data/README.rst @@ -3,9 +3,9 @@ Language-Specific Data This directory contains language-specific data files. Most importantly, you will find here: -1. A list of unique characters for the target language (e.g. English) in `data/alphabet.txt` +1. A list of unique characters for the target language (e.g. English) in ``data/alphabet.txt`` -2. A scorer package (`data/lm/kenlm.scorer`) generated with `data/lm/generate_package.py`. The scorer package includes a binary n-gram language model generated with `data/lm/generate_lm.py`. +2. A scorer package (``data/lm/kenlm.scorer``) generated with ``generate_scorer_package`` (``native_client/generate_scorer_package.cpp``). The scorer package includes a binary n-gram language model generated with ``data/lm/generate_lm.py``. For more information on how to build these resources from scratch, see the ``External scorer scripts`` section on `deepspeech.readthedocs.io `_. diff --git a/doc/Decoder.rst b/doc/Decoder.rst index 63e3ac2d..1115e38e 100644 --- a/doc/Decoder.rst +++ b/doc/Decoder.rst @@ -56,9 +56,11 @@ At decoding time, the scorer is queried every time a Unicode codepoint is predic **Acoustic models trained with ``--utf8`` MUST NOT be used with an alphabet based scorer. Conversely, acoustic models trained with an alphabet file MUST NOT be used with a UTF-8 scorer.** -UTF-8 scorers can be built by using an input corpus with space separated codepoints. If your corpus only contains single codepoints separated by spaces, ``data/lm/generate_package.py`` should automatically enable UTF-8 mode, and it should print the message "Looks like a character based model." +UTF-8 scorers can be built by using an input corpus with space separated codepoints. If your corpus only contains single codepoints separated by spaces, ``generate_scorer_package`` should automatically enable UTF-8 mode, and it should print the message "Looks like a character based model." -If the message "Doesn't look like a character based model." is printed, you should double check your inputs to make sure it only contains single codepoints separated by spaces. UTF-8 mode can be forced by specifying the ``--force_utf8`` flag when running ``data/lm/generate_package.py``, but it is NOT RECOMMENDED. +If the message "Doesn't look like a character based model." is printed, you should double check your inputs to make sure it only contains single codepoints separated by spaces. UTF-8 mode can be forced by specifying the ``--force_utf8`` flag when running ``generate_scorer_package``, but it is NOT RECOMMENDED. + +See :ref:`scorer-scripts` for more details on using ``generate_scorer_package``. Because KenLM uses spaces as a word separator, the resulting language model will not include space characters in it. If you wish to use UTF-8 mode but still model spaces, you need to replace spaces in the input corpus with a different character **before** converting it to space separated codepoints. For example: diff --git a/doc/Scorer.rst b/doc/Scorer.rst index 8df94a74..42a7e833 100644 --- a/doc/Scorer.rst +++ b/doc/Scorer.rst @@ -5,7 +5,9 @@ External scorer scripts DeepSpeech pre-trained models include an external scorer. This document explains how to reproduce our external scorer, as well as adapt the scripts to create your own. -The scorer is composed of two sub-components, a KenLM language model and a trie data structure containing all words in the vocabulary. In order to create the scorer package, first we must create a KenLM language model (using ``data/lm/generate_lm.py``, and then use ``data/lm/generate_package.py`` to create the final package file including the trie data structure. +The scorer is composed of two sub-components, a KenLM language model and a trie data structure containing all words in the vocabulary. In order to create the scorer package, first we must create a KenLM language model (using ``data/lm/generate_lm.py``, and then use ``generate_scorer_package`` to create the final package file including the trie data structure. + +The ``generate_scorer_package`` binary is part of the native client package that is included with official releases. You can find the appropriate archive for your platform in the `GitHub release downloads `_. The native client package is named ``native_client.{arch}.{config}.{plat}.tar.xz``, where ``{arch}`` is the architecture the binary was built for, for example ``amd64`` or ``arm64``, ``config`` is the build configuration, which for building decoder packages does not matter, and ``{plat}`` is the platform the binary was built-for, for example ``linux`` or ``osx``. If you wanted to run the ``generate_scorer_package`` binary on a Linux desktop, you would download ``native_client.amd64.cpu.linux.tar.xz``. Reproducing our external scorer ------------------------------- @@ -36,12 +38,15 @@ Else you have to build `KenLM `_ first and then pa --binary_a_bits 255 --binary_q_bits 8 --binary_type trie -Afterwards you can use ``generate_package.py`` to generate the scorer package using the ``lm.binary`` and ``vocab-500000.txt`` files: +Afterwards you can use ``generate_scorer_package`` to generate the scorer package using the ``lm.binary`` and ``vocab-500000.txt`` files: .. code-block:: bash cd data/lm - python3 generate_package.py --alphabet ../alphabet.txt --lm lm.binary --vocab vocab-500000.txt \ + # Download and extract appropriate native_client package: + curl -LO ... + tar xvf native_client.*.tar.xz + ./generate_scorer_package --alphabet ../alphabet.txt --lm lm.binary --vocab vocab-500000.txt \ --package kenlm.scorer --default_alpha 0.931289039105002 --default_beta 1.1834137581510284 Building your own scorer @@ -51,7 +56,6 @@ Building your own scorer can be useful if you're using models in a narrow usage The LibriSpeech LM training text used by our scorer is around 4GB uncompressed, which should give an idea of the size of a corpus needed for a reasonable language model for general speech recognition. For more constrained use cases with smaller vocabularies, you don't need as much data, but you should still try to gather as much as you can. -With a text corpus in hand, you can then re-use the ``generate_lm.py`` and ``generate_package.py`` scripts to create your own scorer that is compatible with DeepSpeech clients and language bindings. Before building the language model, you must first familiarize yourself with the `KenLM toolkit `_. Most of the options exposed by the ``generate_lm.py`` script are simply forwarded to KenLM options of the same name, so you must read the KenLM documentation in order to fully understand their behavior. +With a text corpus in hand, you can then re-use ``generate_lm.py`` and ``generate_scorer_package`` to create your own scorer that is compatible with DeepSpeech clients and language bindings. Before building the language model, you must first familiarize yourself with the `KenLM toolkit `_. Most of the options exposed by the ``generate_lm.py`` script are simply forwarded to KenLM options of the same name, so you must read the KenLM documentation in order to fully understand their behavior. -After using ``generate_lm.py`` to create a KenLM language model binary file, you can use ``generate_package.py`` to create a scorer package as described in the previous section. Note that we have a :github:`lm_optimizer.py script ` which can be used to find good default values for alpha and beta. To use it, you must first -generate a package with any value set for default alpha and beta flags. For this step, it doesn't matter what values you use, as they'll be overridden by ``lm_optimizer.py``. Then, use ``lm_optimizer.py`` with this scorer file to find good alpha and beta values. Finally, use ``generate_package.py`` again, this time with the new values. +After using ``generate_lm.py`` to create a KenLM language model binary file, you can use ``generate_scorer_package`` to create a scorer package as described in the previous section. Note that we have a :github:`lm_optimizer.py script ` which can be used to find good default values for alpha and beta. To use it, you must first generate a package with any value set for default alpha and beta flags. For this step, it doesn't matter what values you use, as they'll be overridden by ``lm_optimizer.py`` later. Then, use ``lm_optimizer.py`` with this scorer file to find good alpha and beta values. Finally, use ``generate_scorer_package`` again, this time with the new values. diff --git a/training/deepspeech_training/util/flags.py b/training/deepspeech_training/util/flags.py index c31eb461..6bf64251 100644 --- a/training/deepspeech_training/util/flags.py +++ b/training/deepspeech_training/util/flags.py @@ -151,7 +151,7 @@ def create_flags(): f.DEFINE_boolean('utf8', False, 'enable UTF-8 mode. When this is used the model outputs UTF-8 sequences directly rather than using an alphabet mapping.') f.DEFINE_string('alphabet_config_path', 'data/alphabet.txt', 'path to the configuration file specifying the alphabet used by the network. See the comment in data/alphabet.txt for a description of the format.') - f.DEFINE_string('scorer_path', 'data/lm/kenlm.scorer', 'path to the external scorer file created with data/lm/generate_package.py') + f.DEFINE_string('scorer_path', 'data/lm/kenlm.scorer', 'path to the external scorer file.') f.DEFINE_alias('scorer', 'scorer_path') f.DEFINE_integer('beam_width', 1024, 'beam width used in the CTC decoder when building candidate transcriptions') f.DEFINE_float('lm_alpha', 0.931289039105002, 'the alpha hyperparameter of the CTC decoder. Language Model weight.') From d0bd1e5c8eea0021fbc677a9e451fe0b4fc8d910 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 30 Jun 2020 20:01:44 +0200 Subject: [PATCH 16/20] Add more doc text around distinction between various pre-trained model files --- doc/USING.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/USING.rst b/doc/USING.rst index db3013d8..2d874ba1 100644 --- a/doc/USING.rst +++ b/doc/USING.rst @@ -40,6 +40,20 @@ If you want to use the pre-trained English model for performing speech-to-text, wget https://github.com/mozilla/DeepSpeech/releases/download/v0.7.4/deepspeech-0.7.4-models.pbmm wget https://github.com/mozilla/DeepSpeech/releases/download/v0.7.4/deepspeech-0.7.4-models.scorer +There are several pre-trained model files available in official releases. Files ending in ``.pbmm`` are compatible with clients and language bindings built against the standard TensorFlow runtime. Usually these packages are simply called ``deepspeech``. These files are also compatible with CUDA enabled clients and language bindings. These packages are usually called ``deepspeech-gpu``. Files ending in ``.tflite`` are compatible with clients and language bindings built against the `TensorFlow Lite runtime `_. These models are optimized for size and performance in low power devices. On desktop platforms, the compatible packages are called ``deepspeech-tflite``. On Android and Raspberry Pi, we only publish TensorFlow Lite enabled packages, and they are simply called ``deepspeech``. You can see a full list of supported platforms and which TensorFlow runtime is supported at :ref:`supported-platforms-inference`. + ++--------------------+---------------------+---------------------+ +| Package/Model type | .pbmm | .tflite | ++====================+=====================+=====================+ +| deepspeech | Depends on platform | Depends on platform | ++--------------------+---------------------+---------------------+ +| deepspeech-gpu | ✅ | ❌ | ++--------------------+---------------------+---------------------+ +| deepspeech-tflite | ❌ | ✅ | ++--------------------+---------------------+---------------------+ + +Finally, the pre-trained model files also include files ending in ``.scorer``. These are external scorers (language models) that are used at inference time in conjunction with an acoustic model (``.pbmm`` or ``.tflite`` file) to produce transcriptions. We also provide further documentation on :ref:`the decoding process ` and :ref:`how language models are generated `. + Model compatibility ^^^^^^^^^^^^^^^^^^^ From 5c41b8966ef040c0eef022f2bcc9d2d1afdaca50 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Tue, 30 Jun 2020 20:32:21 +0200 Subject: [PATCH 17/20] Fix broken link to C API docs X-DeepSpeech: NOBUILD --- doc/C-API.rst | 2 ++ doc/USING.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/C-API.rst b/doc/C-API.rst index 7713eeb8..e96f3e12 100644 --- a/doc/C-API.rst +++ b/doc/C-API.rst @@ -1,3 +1,5 @@ +.. _c-usage: + C API ===== diff --git a/doc/USING.rst b/doc/USING.rst index 2d874ba1..fe650a07 100644 --- a/doc/USING.rst +++ b/doc/USING.rst @@ -5,7 +5,7 @@ Using a Pre-trained Model Inference using a DeepSpeech pre-trained model can be done with a client/language binding package. We have four clients/language bindings in this repository, listed below, and also a few community-maintained clients/language bindings in other repositories, listed `further down in this README <#third-party-bindings>`_. -* `The C API `. +* :ref:`The C API `. * :ref:`The Python package/language binding ` * :ref:`The Node.JS package/language binding ` * :ref:`The command-line client ` From 65915c7f57a1b39a36e8d5f8e28595251f1bd752 Mon Sep 17 00:00:00 2001 From: Reuben Morais Date: Thu, 2 Jul 2020 14:09:42 +0200 Subject: [PATCH 18/20] Address review comments --- data/README.rst | 2 +- doc/Scorer.rst | 2 +- taskcluster/.shared.yml | 36 ++++++++++++++++++------------------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/data/README.rst b/data/README.rst index aeb34864..3a60ea5a 100644 --- a/data/README.rst +++ b/data/README.rst @@ -3,7 +3,7 @@ Language-Specific Data This directory contains language-specific data files. Most importantly, you will find here: -1. A list of unique characters for the target language (e.g. English) in ``data/alphabet.txt`` +1. A list of unique characters for the target language (e.g. English) in ``data/alphabet.txt``. After installing the training code, you can check ``python -m deepspeech_training.util.check_characters --help`` for a tool that creates an alphabet file from a list of training CSV files. 2. A scorer package (``data/lm/kenlm.scorer``) generated with ``generate_scorer_package`` (``native_client/generate_scorer_package.cpp``). The scorer package includes a binary n-gram language model generated with ``data/lm/generate_lm.py``. diff --git a/doc/Scorer.rst b/doc/Scorer.rst index 42a7e833..04ce2d68 100644 --- a/doc/Scorer.rst +++ b/doc/Scorer.rst @@ -44,7 +44,7 @@ Afterwards you can use ``generate_scorer_package`` to generate the scorer packag cd data/lm # Download and extract appropriate native_client package: - curl -LO ... + curl -LO http://github.com/mozilla/DeepSpeech/releases/... tar xvf native_client.*.tar.xz ./generate_scorer_package --alphabet ../alphabet.txt --lm lm.binary --vocab vocab-500000.txt \ --package kenlm.scorer --default_alpha 0.931289039105002 --default_beta 1.1834137581510284 diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 46cd6983..ca6cd9ff 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -142,32 +142,32 @@ system: namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" tensorflow: linux_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.cpu/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.cpu" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.cpu/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.cpu" linux_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.cuda/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.cuda" linux_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.arm/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.arm" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.arm/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.arm" linux_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.arm64" darwin_amd64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.osx/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.osx" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.osx/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.osx" android_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.android-arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.android-arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.android-arm64" android_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.android-armv7/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.android-armv7" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.android-armv7" win_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.win/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.win" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.win/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.win" win_amd64_cuda: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.win-cuda/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2-submodule.347767452d19a45a6aeb3694e54adce4d945634a.1.win-cuda" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.win-cuda/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.win-cuda" username: 'build-user' homedir: linux: '/home/build-user' From 67004ca1377e22c2479053e82ce6d9c54321affb Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 23 Jul 2019 14:00:08 +0200 Subject: [PATCH 19/20] Enable TFLite delegations --- SUPPORT.rst | 2 + native_client/README.rst => doc/BUILDING.rst | 121 ++++++++++++++++++- doc/index.rst | 4 + native_client/BUILD | 1 + native_client/java/README.rst | 72 ----------- native_client/tflitemodelstate.cc | 79 +++++++++++- native_client/tflitemodelstate.h | 1 + taskcluster/.shared.yml | 12 +- taskcluster/android-build.sh | 2 +- taskcluster/tf_tc-vars.sh | 4 +- 10 files changed, 213 insertions(+), 85 deletions(-) rename native_client/README.rst => doc/BUILDING.rst (61%) delete mode 100644 native_client/java/README.rst diff --git a/SUPPORT.rst b/SUPPORT.rst index f2291e01..d72a7418 100644 --- a/SUPPORT.rst +++ b/SUPPORT.rst @@ -1,3 +1,5 @@ +.. _support: + Contact/Getting Help ==================== diff --git a/native_client/README.rst b/doc/BUILDING.rst similarity index 61% rename from native_client/README.rst rename to doc/BUILDING.rst index fb1c2d0e..3c20cb29 100644 --- a/native_client/README.rst +++ b/doc/BUILDING.rst @@ -1,7 +1,11 @@ +.. _build-native-client: Building DeepSpeech Binaries ============================ +This section describes how to rebuild binaries. We have already several prebuilt binaries for all the supported platform, +it is highly advised to use them except if you know what you are doing. + If you'd like to build the DeepSpeech binaries yourself, you'll need the following pre-requisites downloaded and installed: * `Bazel 2.0.0 `_ @@ -165,13 +169,31 @@ The path of the system tree can be overridden from the default values defined in cd ../DeepSpeech/native_client make TARGET= deepspeech -Android devices -^^^^^^^^^^^^^^^ +Android devices support +----------------------- -We have preliminary support for Android relying on TensorFlow Lite, with Java and JNI bindinds. For more details on how to experiment with those, please refer to ``native_client/java/README.rst``. +We have support for Android relying on TensorFlow Lite, with Java and JNI bindinds. For more details on how to experiment with those, please refer to the section below. Please refer to TensorFlow documentation on how to setup the environment to build for Android (SDK and NDK required). +Using the library from Android project +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We provide uptodate and tested ``libdeepspeech`` usable as an ``AAR`` package, +for Android versions starting with 7.0 to 11.0. The package is published on +`JCenter `_, +and the ``JCenter`` repository should be available by default in any Android +project. Please make sure your project is setup to pull from this repository. +You can then include the library by just adding this line to your +``gradle.build``, adjusting ``VERSION`` to the version you need: + +.. code-block:: + + implementation 'deepspeech.mozilla.org:libdeepspeech:VERSION@aar' + +Building ``libdeepspeech.so`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + You can build the ``libdeepspeech.so`` using (ARMv7): .. code-block:: @@ -184,6 +206,25 @@ Or (ARM64): bazel build --workspace_status_command="bash native_client/bazel_workspace_status_cmd.sh" --config=monolithic --config=android --config=android_arm64 --define=runtime=tflite --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 //native_client:libdeepspeech.so +Building ``libdeepspeech.aar`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the unlikely event you have to rebuild the JNI bindings, source code is +available under the ``libdeepspeech`` subdirectory. Building depends on shared +object: please ensure to place ``libdeepspeech.so`` into the +``libdeepspeech/libs/{arm64-v8a,armeabi-v7a,x86_64}/`` matching subdirectories. + +Building the bindings is managed by ``gradle`` and should be limited to issuing +``./gradlew libdeepspeech:build``, producing an ``AAR`` package in +``./libdeepspeech/build/outputs/aar/``. + +Please note that you might have to copy the file to a local Maven repository +and adapt file naming (when missing, the error message should states what +filename it expects and where). + +Building C++ ``deepspeech`` binary +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Building the ``deepspeech`` binary will happen through ``ndk-build`` (ARMv7): .. code-block:: @@ -197,3 +238,77 @@ And (ARM64): cd ../DeepSpeech/native_client $ANDROID_NDK_HOME/ndk-build APP_PLATFORM=android-21 APP_BUILD_SCRIPT=$(pwd)/Android.mk NDK_PROJECT_PATH=$(pwd) APP_STL=c++_shared TFDIR=$(pwd)/../tensorflow/ TARGET_ARCH_ABI=arm64-v8a + +Android demo APK +^^^^^^^^^^^^^^^^ + +Provided is a very simple Android demo app that allows you to test the library. +You can build it with ``make apk`` and install the resulting APK file. Please +refer to Gradle documentation for more details. + +The ``APK`` should be produced in ``/app/build/outputs/apk/``. This demo app might +require external storage permissions. You can then push models files to your +device, set the path to the file in the UI and try to run on an audio file. +When running, it should first play the audio file and then run the decoding. At +the end of the decoding, you should be presented with the decoded text as well +as time elapsed to decode in miliseconds. + +This application is very limited on purpose, and is only here as a very basic +demo of one usage of the application. For example, it's only able to read PCM +mono 16kHz 16-bits file and it might fail on some WAVE file that are not +following exactly the specification. + +Running ``deepspeech`` via adb +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You should use ``adb push`` to send data to device, please refer to Android +documentation on how to use that. + +Please push DeepSpeech data to ``/sdcard/deepspeech/``\ , including: + + +* ``output_graph.tflite`` which is the TF Lite model +* ``kenlm.scorer``, if you want to use the scorer; please be aware that too big + scorer will make the device run out of memory + +Then, push binaries from ``native_client.tar.xz`` to ``/data/local/tmp/ds``\ : + +* ``deepspeech`` +* ``libdeepspeech.so`` +* ``libc++_shared.so`` + +You should then be able to run as usual, using a shell from ``adb shell``\ : + +.. code-block:: + + user@device$ cd /data/local/tmp/ds/ + user@device$ LD_LIBRARY_PATH=$(pwd)/ ./deepspeech [...] + +Please note that Android linker does not support ``rpath`` so you have to set +``LD_LIBRARY_PATH``. Properly wrapped / packaged bindings does embed the library +at a place the linker knows where to search, so Android apps will be fine. + +Delegation API +^^^^^^^^^^^^^^ + +TensorFlow Lite supports Delegate API to offload some computation from the main +CPU. Please refer to `TensorFlow's documentation +`_ for details. + +To ease with experimentations, we have enabled some of those delegations on our +Android builds: * GPU, to leverage OpenGL capabilities * NNAPI, the Android API +to leverage GPU / DSP / NPU * Hexagon, the Qualcomm-specific DSP + +This is highly experimental: + +* Requires passing environment variable ``DS_TFLITE_DELEGATE`` with values of + ``gpu``, ``nnapi`` or ``hexagon`` (only one at a time) +* Might require exported model changes (some Op might not be supported) +* We can't guarantee it will work, nor it will be faster than default + implementation + +Feedback on improving this is welcome: how it could be exposed in the API, how +much performance gains do you get in your applications, how you had to change +the model to make it work with a delegate, etc. + +See :ref:`the support / contact details ` diff --git a/doc/index.rst b/doc/index.rst index 659439ee..e8991d3f 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -60,6 +60,10 @@ See the output of ``deepspeech -h`` for more information on the use of ``deepspe SUPPORTED_PLATFORMS + BUILDING + +.. include:: ../SUPPORT.rst + .. toctree:: :maxdepth: 2 :caption: Decoder and scorer diff --git a/native_client/BUILD b/native_client/BUILD index 232d99c7..b38979e5 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -139,6 +139,7 @@ tf_cc_shared_object( deps = select({ "//native_client:tflite": [ "//tensorflow/lite/kernels:builtin_ops", + "//tensorflow/lite/tools/evaluation:utils", ], "//conditions:default": [ "//tensorflow/core:core_cpu", diff --git a/native_client/java/README.rst b/native_client/java/README.rst deleted file mode 100644 index 626400d0..00000000 --- a/native_client/java/README.rst +++ /dev/null @@ -1,72 +0,0 @@ - -DeepSpeech Java / Android bindings -================================== - -This is still preliminary work. Please refer to ``native_client/README.rst`` for -building ``libdeepspeech.so`` and ``deepspeech`` binary for Android on ARMv7 and -ARM64 arch. - -Android Java / JNI bindings: ``libdeepspeech`` -================================================== - -Java / JNI bindings are available under the ``libdeepspeech`` subdirectory. -Building depends on prebuilt shared object. Please ensure to place -``libdeepspeech.so`` into the ``libdeepspeech/libs/{arm64-v8a,armeabi-v7a}/`` -matching subdirectories. - -Building the bindings is managed by ``gradle`` and should be limited to issuing -``./gradlew libdeepspeech:build``\ , producing an ``AAR`` package in -``./libdeepspeech/build/outputs/aar/``. This can later be used by other -Gradle-based build with the following configuration: - -.. code-block:: - - implementation 'deepspeech.mozilla.org:libdeepspeech:VERSION@aar' - -Please note that you might have to copy the file to a local Maven repository -and adapt file naming (when missing, the error message should states what -filename it expects and where). - -Android demo APK -================ - -Provided is a very simple Android demo app that allows you to test the library. -You can build it with ``make apk`` and install the resulting APK file. Please -refer to Gradle documentation for more details. - -The ``APK`` should be produced in ``/app/build/outputs/apk/``. This demo app might -require external storage permissions. You can then push models files to your -device, set the path to the file in the UI and try to run on an audio file. -When running, it should first play the audio file and then run the decoding. At -the end of the decoding, you should be presented with the decoded text as well -as time elapsed to decode in miliseconds. - -Running ``deepspeech`` via adb -================================== - -You should use ``adb push`` to send data to device, please refer to Android -documentation on how to use that. - -Please push DeepSpeech data to ``/sdcard/deepspeech/``\ , including: - - -* ``output_graph.tflite`` which is the TF Lite model -* ``kenlm.scorer``, if you want to use the scorer; please be aware that too big - scorer will make the device run out of memory - -Then, push binaries from ``native_client.tar.xz`` to ``/data/local/tmp/ds``\ : - -* ``deepspeech`` -* ``libdeepspeech.so`` -* ``libc++_shared.so`` - -You should then be able to run as usual, using a shell from ``adb shell``\ : - -.. code-block:: - - user@device$ cd /data/local/tmp/ds/ - user@device$ LD_LIBRARY_PATH=$(pwd)/ ./deepspeech [...] - -Please note that Android linker does not support ``rpath`` so you have to set -``LD_LIBRARY_PATH``. Properly wrapped / packaged bindings does embed the library -at a place the linker knows where to search, so Android apps will be fine. diff --git a/native_client/tflitemodelstate.cc b/native_client/tflitemodelstate.cc index 12be0d37..50a68a4b 100644 --- a/native_client/tflitemodelstate.cc +++ b/native_client/tflitemodelstate.cc @@ -1,8 +1,17 @@ #include "tflitemodelstate.h" - #include "tensorflow/lite/string_util.h" #include "workspace_status.h" +#ifdef __ANDROID__ +#include +#define LOG_TAG "libdeepspeech" +#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) +#else +#define LOGD(...) +#define LOGE(...) +#endif // __ANDROID__ + using namespace tflite; using std::vector; @@ -90,6 +99,62 @@ TFLiteModelState::~TFLiteModelState() { } +std::map +getTfliteDelegates() +{ + std::map delegates; + + const char* env_delegate_c = std::getenv("DS_TFLITE_DELEGATE"); + std::string env_delegate = (env_delegate_c != nullptr) ? env_delegate_c : ""; + +#ifdef __ANDROID__ + if (env_delegate == std::string("gpu")) { + LOGD("Trying to get GPU delegate ..."); + // Try to get GPU delegate + { + tflite::Interpreter::TfLiteDelegatePtr delegate = evaluation::CreateGPUDelegate(); + if (!delegate) { + LOGD("GPU delegation not supported"); + } else { + LOGD("GPU delegation supported"); + delegates.emplace("GPU", std::move(delegate)); + } + } + } + + if (env_delegate == std::string("nnapi")) { + LOGD("Trying to get NNAPI delegate ..."); + // Try to get Android NNAPI delegate + { + tflite::Interpreter::TfLiteDelegatePtr delegate = evaluation::CreateNNAPIDelegate(); + if (!delegate) { + LOGD("NNAPI delegation not supported"); + } else { + LOGD("NNAPI delegation supported"); + delegates.emplace("NNAPI", std::move(delegate)); + } + } + } + + if (env_delegate == std::string("hexagon")) { + LOGD("Trying to get Hexagon delegate ..."); + // Try to get Android Hexagon delegate + { + const std::string libhexagon_path("/data/local/tmp"); + tflite::Interpreter::TfLiteDelegatePtr delegate = evaluation::CreateHexagonDelegate(libhexagon_path, /* profiler */ false); + if (!delegate) { + LOGD("Hexagon delegation not supported"); + } else { + LOGD("Hexagon delegation supported"); + delegates.emplace("Hexagon", std::move(delegate)); + } + } + } +#endif // __ANDROID__ + + return delegates; +} + int TFLiteModelState::init(const char* model_path) { @@ -111,9 +176,21 @@ TFLiteModelState::init(const char* model_path) return DS_ERR_FAIL_INTERPRETER; } + LOGD("Trying to detect delegates ..."); + std::map delegates = getTfliteDelegates(); + LOGD("Finished enumerating delegates ..."); + interpreter_->AllocateTensors(); interpreter_->SetNumThreads(4); + LOGD("Trying to use delegates ..."); + for (const auto& delegate : delegates) { + LOGD("Trying to apply delegate %s", delegate.first.c_str()); + if (interpreter_->ModifyGraphWithDelegate(delegate.second.get()) != kTfLiteOk) { + LOGD("FAILED to apply delegate %s to the graph", delegate.first.c_str()); + } + } + // Query all the index once input_node_idx_ = get_input_tensor_by_name("input_node"); previous_state_c_idx_ = get_input_tensor_by_name("previous_state_c"); diff --git a/native_client/tflitemodelstate.h b/native_client/tflitemodelstate.h index 11532e64..ace62ecf 100644 --- a/native_client/tflitemodelstate.h +++ b/native_client/tflitemodelstate.h @@ -6,6 +6,7 @@ #include "tensorflow/lite/model.h" #include "tensorflow/lite/kernels/register.h" +#include "tensorflow/lite/tools/evaluation/utils.h" #include "modelstate.h" diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index ca6cd9ff..076a8061 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -142,8 +142,8 @@ system: namespace: "project.deepspeech.swig.win.amd64.b5fea54d39832d1d132d7dd921b69c0c2c9d5118" tensorflow: linux_amd64_cpu: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.cpu/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.cpu" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.1.cpu/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.1.cpu" linux_amd64_cuda: url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.cuda/artifacts/public/home.tar.xz" namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.cuda" @@ -157,11 +157,11 @@ system: url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.osx/artifacts/public/home.tar.xz" namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.osx" android_arm64: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.android-arm64/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.android-arm64" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1.1.bf55d362bb11e973b8f5.1.a3e5bf44d.1.android-arm64/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1.1.bf55d362bb11e973b8f5.1.a3e5bf44d.1.android-arm64" android_armv7: - url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.android-armv7/artifacts/public/home.tar.xz" - namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.android-armv7" + url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1.1.bf55d362bb11e973b8f5.1.a3e5bf44d.1.android-armv7/artifacts/public/home.tar.xz" + namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1.1.bf55d362bb11e973b8f5.1.a3e5bf44d.1.android-armv7" win_amd64_cpu: url: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.win/artifacts/public/home.tar.xz" namespace: "project.deepspeech.tensorflow.pip.r2.2.518c1d04bf55d362bb11e973b8f5d0aa3e5bf44d.0.win" diff --git a/taskcluster/android-build.sh b/taskcluster/android-build.sh index 83d8eecb..2b9e0393 100644 --- a/taskcluster/android-build.sh +++ b/taskcluster/android-build.sh @@ -22,7 +22,7 @@ if [ "${arm_flavor}" = "arm64-v8a" ]; then fi if [ "${arm_flavor}" = "x86_64" ]; then - LOCAL_ANDROID_FLAGS="--config=android --cpu=x86_64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99" + LOCAL_ANDROID_FLAGS="--config=android --cpu=x86_64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99" fi BAZEL_BUILD_FLAGS="--define=runtime=tflite ${LOCAL_ANDROID_FLAGS} ${BAZEL_EXTRA_FLAGS}" diff --git a/taskcluster/tf_tc-vars.sh b/taskcluster/tf_tc-vars.sh index c6aa1746..8150bb8d 100755 --- a/taskcluster/tf_tc-vars.sh +++ b/taskcluster/tf_tc-vars.sh @@ -168,8 +168,8 @@ else fi BAZEL_ARM_FLAGS="--config=rpi3 --config=rpi3_opt --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_ARM64_FLAGS="--config=rpi3-armv8 --config=rpi3-armv8_opt --copt=-DTFLITE_WITH_RUY_GEMV" -BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" -BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++11 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" +BAZEL_ANDROID_ARM_FLAGS="--config=android --config=android_arm --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" +BAZEL_ANDROID_ARM64_FLAGS="--config=android --config=android_arm64 --action_env ANDROID_NDK_API_LEVEL=21 --cxxopt=-std=c++14 --copt=-D_GLIBCXX_USE_C99 --copt=-DTFLITE_WITH_RUY_GEMV" BAZEL_CUDA_FLAGS="--config=cuda" if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then From 436561b0e411607924a1be381f09ab87573d732d Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 30 Jun 2020 11:56:40 +0200 Subject: [PATCH 20/20] Support for Docker Hub automated builds --- hooks/pre_build | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 hooks/pre_build diff --git a/hooks/pre_build b/hooks/pre_build new file mode 100755 index 00000000..27b204ff --- /dev/null +++ b/hooks/pre_build @@ -0,0 +1,34 @@ +#!/bin/bash +# +# This script is intended for Docker Hub Automated Builds since we can't +# build on TaskCluster because of too old Docker version (see issue #3057) +# +# Docker Hub expects an existing ready-to-use Dockerfile, and this hook allows +# to generate one basing on the existing |make| template, but we need to change +# in-place, hence why the |cp| statement +# +# Docker Hub Automated Builds needs to be configured as: +# +# Source Type | Source | Docker Tag | Dockerfile Location | Build Context +# ---------------------------------------------------------------------------------------- +# Tag | /^v([0-9.\-a-zA-Z]+)/ | v{\1} | Dockerfile.train.tmpl | / +# Tag | /^v([0-9.\-a-zA-Z]+)/ | v{\1} | Dockerfile.build.tmpl | / +# +# Docker Hub Automated builds will populate some env variables: +# - DOCKERFILE_PATH: the filename defined as "Dockerfile Location" +# - SOURCE_REPOSITORY_URL: the url of the repo triggering the tag event +# - SOURCE_COMMIT: the sha1 of the commit triggering the tag event +# +# More details: https://docs.docker.com/docker-hub/builds/advanced/#environment-variables-for-building-and-testing + +set -e # Exit immediately if a command exits with a non-zero status. +set -u # Treat unset variables as an error. + +DOCKERFILE_TARGET=$(echo ${DOCKERFILE_PATH} | sed -e 's/\.tmpl//g') +make DEEPSPEECH_REPO=${SOURCE_REPOSITORY_URL} DEEPSPEECH_SHA=${SOURCE_COMMIT} ${DOCKERFILE_TARGET} +cp ${DOCKERFILE_TARGET} ${DOCKERFILE_PATH} + +#### If you need to inspect the generated Dockerfile +## echo "----8<----8<----8<----8<----8<----8<----8<----8<----8<----8<----8<----8<----" +## cat ${DOCKERFILE_PATH} +## echo "----8<----8<----8<----8<----8<----8<----8<----8<----8<----8<----8<----8<----"