experiments
/
STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Build DeepSpeech using TensorFlow r2.2

This commit is contained in:
Alexandre Lissy 2020-06-02 14:06:38 +02:00
parent 41d7b4e6f0
commit bc086ec998
17 changed files with 43 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Dockerfile.build.tmpl
View File

@ -1,7 +1,7 @@
# Please refer to the USING documentation, "Dockerfile for building from source" # Please refer to the USING documentation, "Dockerfile for building from source"
# Need devel version cause we need /usr/include/cudnn.h # Need devel version cause we need /usr/include/cudnn.h
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04
ENV DEEPSPEECH_REPO=#DEEPSPEECH_REPO# ENV DEEPSPEECH_REPO=#DEEPSPEECH_REPO#
ENV DEEPSPEECH_SHA=#DEEPSPEECH_SHA# ENV DEEPSPEECH_SHA=#DEEPSPEECH_SHA#
@ -46,7 +46,7 @@ RUN update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
# Install Bazel # Install Bazel
RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/0.24.1/bazel_0.24.1-linux-x86_64.deb" RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel_2.0.0-linux-x86_64.deb"
RUN dpkg -i bazel_*.deb RUN dpkg -i bazel_*.deb
# << END Install base software # << END Install base software
@ -56,12 +56,15 @@ RUN dpkg -i bazel_*.deb
# Clone TensorFlow from Mozilla repo # Clone TensorFlow from Mozilla repo
RUN git clone https://github.com/mozilla/tensorflow/ RUN git clone https://github.com/mozilla/tensorflow/
WORKDIR /tensorflow WORKDIR /tensorflow
RUN git checkout r1.15 RUN git checkout r2.2
# GPU Environment Setup # GPU Environment Setup
ENV TF_NEED_ROCM 0
ENV TF_NEED_OPENCL_SYCL 0
ENV TF_NEED_OPENCL 0
ENV TF_NEED_CUDA 1 ENV TF_NEED_CUDA 1
ENV TF_CUDA_PATHS "/usr,/usr/local/cuda,/usr/lib/x86_64-linux-gnu/" ENV TF_CUDA_PATHS "/usr,/usr/local/cuda-10.1,/usr/lib/x86_64-linux-gnu/"
ENV TF_CUDA_VERSION 10.0 ENV TF_CUDA_VERSION 10.1
ENV TF_CUDNN_VERSION 7.6 ENV TF_CUDNN_VERSION 7.6
ENV TF_CUDA_COMPUTE_CAPABILITIES 6.0 ENV TF_CUDA_COMPUTE_CAPABILITIES 6.0
ENV TF_NCCL_VERSION 2.4 ENV TF_NCCL_VERSION 2.4
@ -94,7 +97,7 @@ ENV TF_NEED_VERBS 0
ENV TF_NEED_OPENCL_SYCL 0 ENV TF_NEED_OPENCL_SYCL 0
ENV PYTHON_BIN_PATH /usr/bin/python3.6 ENV PYTHON_BIN_PATH /usr/bin/python3.6
ENV PYTHON_LIB_PATH /usr/lib/python3.6/dist-packages ENV PYTHON_LIB_PATH /usr/local/lib/python3.6/dist-packages
# << END Configure Tensorflow Build # << END Configure Tensorflow Build

2
doc/USING.rst
View File

@ -28,7 +28,7 @@ Please refer to your system's documentation on how to install these dependencies
CUDA dependency CUDA dependency
^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^
The GPU capable builds (Python, NodeJS, C++, etc) depend on the same CUDA runtime as upstream TensorFlow. Currently with TensorFlow 1.15 it depends on CUDA 10.0 and CuDNN v7.6. `See the TensorFlow documentation <https://www.tensorflow.org/install/gpu>`_. The GPU capable builds (Python, NodeJS, C++, etc) depend on the same CUDA runtime as upstream TensorFlow. Currently with TensorFlow 2.2 it depends on CUDA 10.1 and CuDNN v7.6. `See the TensorFlow documentation <https://www.tensorflow.org/install/gpu>`_.
Getting the pre-trained model Getting the pre-trained model
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

12
native_client/BUILD
View File

@ -1,10 +1,8 @@
# Description: Deepspeech native client library. # Description: Deepspeech native client library.
load( load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_cc_shared_object")
"@org_tensorflow//tensorflow:tensorflow.bzl", load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
"if_cuda",
"tf_cc_shared_object",
)
load( load(
"@org_tensorflow//tensorflow/lite:build_def.bzl", "@org_tensorflow//tensorflow/lite:build_def.bzl",
"tflite_copts", "tflite_copts",
@ -143,7 +141,6 @@ tf_cc_shared_object(
### CPU only build, libdeepspeech.so file size reduced by ~50% ### CPU only build, libdeepspeech.so file size reduced by ~50%
"//tensorflow/core/kernels:spectrogram_op", # AudioSpectrogram "//tensorflow/core/kernels:spectrogram_op", # AudioSpectrogram
"//tensorflow/core/kernels:bias_op", # BiasAdd "//tensorflow/core/kernels:bias_op", # BiasAdd
"//tensorflow/contrib/rnn:lstm_ops_kernels", # BlockLSTM
"//tensorflow/core/kernels:cast_op", # Cast "//tensorflow/core/kernels:cast_op", # Cast
"//tensorflow/core/kernels:concat_op", # ConcatV2 "//tensorflow/core/kernels:concat_op", # ConcatV2
"//tensorflow/core/kernels:constant_op", # Const, Placeholder "//tensorflow/core/kernels:constant_op", # Const, Placeholder
@ -163,9 +160,10 @@ tf_cc_shared_object(
"//tensorflow/core/kernels:softmax_op", # Softmax "//tensorflow/core/kernels:softmax_op", # Softmax
"//tensorflow/core/kernels:tile_ops", # Tile "//tensorflow/core/kernels:tile_ops", # Tile
"//tensorflow/core/kernels:transpose_op", # Transpose "//tensorflow/core/kernels:transpose_op", # Transpose
"//tensorflow/core/kernels:rnn_ops", # BlockLSTM
# And we also need the op libs for these ops used in the model: # And we also need the op libs for these ops used in the model:
"//tensorflow/core:audio_ops_op_lib", # AudioSpectrogram, Mfcc "//tensorflow/core:audio_ops_op_lib", # AudioSpectrogram, Mfcc
"//tensorflow/contrib/rnn:lstm_ops_op_lib", # BlockLSTM "//tensorflow/core:rnn_ops_op_lib", # BlockLSTM
"//tensorflow/core:math_ops_op_lib", # Cast, Less, Max, MatMul, Minimum, Range "//tensorflow/core:math_ops_op_lib", # Cast, Less, Max, MatMul, Minimum, Range
"//tensorflow/core:array_ops_op_lib", # ConcatV2, Const, ExpandDims, Fill, GatherNd, Identity, Pack, Placeholder, Reshape, Tile, Transpose "//tensorflow/core:array_ops_op_lib", # ConcatV2, Const, ExpandDims, Fill, GatherNd, Identity, Pack, Placeholder, Reshape, Tile, Transpose
"//tensorflow/core:no_op_op_lib", # NoOp "//tensorflow/core:no_op_op_lib", # NoOp

8
native_client/README.rst
View File

@ -5,8 +5,8 @@ Building DeepSpeech Binaries
If you'd like to build the DeepSpeech binaries yourself, you'll need the following pre-requisites downloaded and installed: If you'd like to build the DeepSpeech binaries yourself, you'll need the following pre-requisites downloaded and installed:
* `Mozilla's TensorFlow r1.15 branch <https://github.com/mozilla/tensorflow/tree/r1.15>`_ * `Mozilla's TensorFlow r2.2 branch <https://github.com/mozilla/tensorflow/tree/r2.2>`_
* `Bazel 0.24.1 <https://github.com/bazelbuild/bazel/releases/tag/0.24.1>`_ * `Bazel 2.0.0 <https://github.com/bazelbuild/bazel/releases/tag/2.0.0>`_
* `General TensorFlow requirements <https://www.tensorflow.org/install/install_sources>`_ * `General TensorFlow requirements <https://www.tensorflow.org/install/install_sources>`_
* `libsox <https://sourceforge.net/projects/sox/>`_ * `libsox <https://sourceforge.net/projects/sox/>`_
@ -36,12 +36,12 @@ Clone our fork of TensorFlow and checkout the correct version:
.. code-block:: .. code-block::
git clone https://github.com/mozilla/tensorflow.git git clone https://github.com/mozilla/tensorflow.git
git checkout origin/r1.15 git checkout origin/r2.2
Bazel: Download & Install Bazel: Download & Install
^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^
First, install Bazel 0.24.1 following the `Bazel installation documentation <https://docs.bazel.build/versions/0.24.0/install.html>`_. First, install Bazel 2.0.0 following the `Bazel installation documentation <https://docs.bazel.build/versions/2.0.0/install.html>`_.
TensorFlow: Configure with Bazel TensorFlow: Configure with Bazel
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

8
native_client/ctcdecode/Makefile
View File

@ -45,15 +45,15 @@ workspace_status.cc:
# variables over several runs # variables over several runs
bindings: clean-keep-third-party workspace_status.cc ds-swig bindings: clean-keep-third-party workspace_status.cc ds-swig
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
find temp_build -type f -name "*.o" -delete find temp_build -type f -name "*.o" -delete
AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
rm -rf temp_build rm -rf temp_build
bindings-debug: clean-keep-third-party workspace_status.cc ds-swig bindings-debug: clean-keep-third-party workspace_status.cc ds-swig
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
$(GENERATE_DEBUG_SYMS) $(GENERATE_DEBUG_SYMS)
find temp_build -type f -name "*.o" -delete find temp_build -type f -name "*.o" -delete
AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
rm -rf temp_build rm -rf temp_build

2
native_client/definitions.mk
View File

@ -48,7 +48,7 @@ endif
endif endif
ifeq ($(TARGET),host-win) ifeq ($(TARGET),host-win)
TOOLCHAIN := '$(VCINSTALLDIR)\bin\amd64\' TOOLCHAIN := '$(VCToolsInstallDir)\bin\Hostx64\x64\'
TOOL_CC := cl.exe TOOL_CC := cl.exe
TOOL_CXX := cl.exe TOOL_CXX := cl.exe
TOOL_LD := link.exe TOOL_LD := link.exe

8
native_client/dotnet/README.rst
View File

@ -30,11 +30,11 @@ Prerequisites
* Windows 10 * Windows 10
* `Windows 10 SDK <https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk>`_ * `Windows 10 SDK <https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk>`_
* `Visual Studio 2017 Community <https://visualstudio.microsoft.com/vs/community/>`_ * `Visual Studio 2019 Community <https://visualstudio.microsoft.com/vs/community/>`_
* `Git Large File Storage <https://git-lfs.github.com/>`_ * `Git Large File Storage <https://git-lfs.github.com/>`_
* `TensorFlow Windows pre-requisites <https://www.tensorflow.org/install/source_windows>`_ * `TensorFlow Windows pre-requisites <https://www.tensorflow.org/install/source_windows>`_
Inside the Visual Studio Installer enable ``MS Build Tools`` and ``VC++ 2015.3 v14.00 (v140) toolset for desktop``. Inside the Visual Studio Installer enable ``MS Build Tools`` and ``VC++ 2019 v16.00 (v160) toolset for desktop``.
If you want to enable CUDA support you need to follow the steps in `the TensorFlow docs for building on Windows with CUDA <https://www.tensorflow.org/install/gpu#windows_setup>`_. If you want to enable CUDA support you need to follow the steps in `the TensorFlow docs for building on Windows with CUDA <https://www.tensorflow.org/install/gpu#windows_setup>`_.
@ -51,7 +51,7 @@ We need to clone ``mozilla/DeepSpeech`` and ``mozilla/tensorflow``.
.. code-block:: bash .. code-block:: bash
git clone --branch r1.15 https://github.com/mozilla/tensorflow git clone --branch r2.2 https://github.com/mozilla/tensorflow
Configuring the paths Configuring the paths
--------------------- ---------------------
@ -113,7 +113,7 @@ If you run CUDA enabled ``native_client`` we need to add the following to the ``
.. code-block:: .. code-block::
C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0\bin C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1\bin
Building the native_client Building the native_client
^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^

4
native_client/javascript/Makefile
View File

@ -41,10 +41,6 @@ package.json: package.json.in
package.json.in > package.json && cat package.json package.json.in > package.json && cat package.json
npm-dev: package.json npm-dev: package.json
ifeq ($(findstring _NT,$(OS)),_NT)
# node-gyp@5.x behaves erratically with VS2015 and MSBuild.exe detection
$(NPM_TOOL) install node-gyp@4.x
endif
$(NPM_TOOL) install --prefix=$(NPM_ROOT)/../ --ignore-scripts --force --verbose --production=false . $(NPM_TOOL) install --prefix=$(NPM_ROOT)/../ --ignore-scripts --force --verbose --production=false .
configure: deepspeech_wrap.cxx package.json npm-dev configure: deepspeech_wrap.cxx package.json npm-dev

2
native_client/javascript/package.json.in
View File

@ -40,7 +40,7 @@
}, },
"devDependencies": { "devDependencies": {
"electron": "^1.7.9", "electron": "^1.7.9",
"node-gyp": "4.x - 5.x", "node-gyp": "5.x",
"typescript": "3.8.x", "typescript": "3.8.x",
"typedoc": "0.17.x", "typedoc": "0.17.x",
"@types/argparse": "1.0.x", "@types/argparse": "1.0.x",

4
native_client/python/Makefile
View File

@ -10,7 +10,7 @@ bindings-clean:
# variables over several runs # variables over several runs
bindings-build: ds-swig bindings-build: ds-swig
pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0
PATH=$(TOOLCHAIN):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) DISTUTILS_USE_SDK=1 PATH=$(TOOLCHAIN):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME)
MANIFEST.in: bindings-build MANIFEST.in: bindings-build
> $@ > $@
@ -21,6 +21,6 @@ MANIFEST.in: bindings-build
bindings-package: MANIFEST.in bindings-package: MANIFEST.in
cat MANIFEST.in cat MANIFEST.in
rm -f temp_build/*_wrap.o temp_build/Release/*_wrap.obj rm -f temp_build/*_wrap.o temp_build/Release/*_wrap.obj
AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS)
bindings: bindings-build bindings-package bindings: bindings-build bindings-package

2
native_client/tfmodelstate.cc
View File

@ -118,7 +118,7 @@ TFModelState::init(const char* model_path)
int beam_width = metadata_outputs[3].scalar<int>()(); int beam_width = metadata_outputs[3].scalar<int>()();
beam_width_ = (unsigned int)(beam_width); beam_width_ = (unsigned int)(beam_width);
string serialized_alphabet = metadata_outputs[4].scalar<string>()(); string serialized_alphabet = metadata_outputs[4].scalar<tensorflow::tstring>()();
err = alphabet_.deserialize(serialized_alphabet.data(), serialized_alphabet.size()); err = alphabet_.deserialize(serialized_alphabet.data(), serialized_alphabet.size());
if (err != 0) { if (err != 0) {
return DS_ERR_INVALID_ALPHABET; return DS_ERR_INVALID_ALPHABET;

2
taskcluster/.build.yml
View File

@ -9,7 +9,7 @@ build:
dependencies: [] dependencies: []
routes: [] routes: []
maxRunTime: 3600 maxRunTime: 3600
docker_image: "ubuntu:14.04" docker_image: "ubuntu:16.04"
system_setup: system_setup:
> >
true true

6
taskcluster/linux-opt-base.tyml
View File

@ -30,7 +30,7 @@ then:
payload: payload:
maxRunTime: { $eval: to_int(build.maxRunTime) } maxRunTime: { $eval: to_int(build.maxRunTime) }
image: "ubuntu:14.04" image: "ubuntu:16.04"
env: env:
TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow}
@ -43,10 +43,8 @@ then:
extraSystemSetup: { $eval: strip(str(build.system_setup)) } extraSystemSetup: { $eval: strip(str(build.system_setup)) }
extraSystemConfig: { $eval: strip(str(build.system_config)) } extraSystemConfig: { $eval: strip(str(build.system_config)) }
in: > in: >
apt-get -qq -y remove --purge ubuntu-advantage-tools &&
${aptEc2Mirrors} &&
adduser --system --home ${system.homedir.linux} ${system.username} && adduser --system --home ${system.homedir.linux} ${system.username} &&
apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_trusty.apt} pixz pkg-config realpath unzip wget zip && ${extraSystemSetup} && apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_trusty.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} &&
cd ${system.homedir.linux}/ && cd ${system.homedir.linux}/ &&
echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh &&
sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} &&

4
taskcluster/tc-build-utils.sh
View File

@ -231,7 +231,7 @@ do_deepspeech_netframework_build()
# Setup dependencies # Setup dependencies
nuget install DeepSpeechConsole/packages.config -OutputDirectory packages/ nuget install DeepSpeechConsole/packages.config -OutputDirectory packages/
MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin\MSBuild.exe')" MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin\MSBuild.exe')"
# We need MSYS2_ARG_CONV_EXCL='/' otherwise the '/' of CLI parameters gets mangled and disappears # We need MSYS2_ARG_CONV_EXCL='/' otherwise the '/' of CLI parameters gets mangled and disappears
# We build the .NET Client for .NET Framework v4.5,v4.6,v4.7 # We build the .NET Client for .NET Framework v4.5,v4.6,v4.7
@ -270,7 +270,7 @@ do_deepspeech_netframework_wpf_example_build()
# Setup dependencies # Setup dependencies
nuget install DeepSpeechWPF/packages.config -OutputDirectory DeepSpeechWPF/packages/ nuget install DeepSpeechWPF/packages.config -OutputDirectory DeepSpeechWPF/packages/
MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin\MSBuild.exe')" MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin\MSBuild.exe')"
# We need MSYS2_ARG_CONV_EXCL='/' otherwise the '/' of CLI parameters gets mangled and disappears # We need MSYS2_ARG_CONV_EXCL='/' otherwise the '/' of CLI parameters gets mangled and disappears
# Build WPF example # Build WPF example

2
taskcluster/test-win-cuda-opt-base.tyml
View File

@ -44,7 +44,7 @@ then:
extraSystemSetup: { $eval: strip(str(build.system_setup)) } extraSystemSetup: { $eval: strip(str(build.system_setup)) }
in: > in: >
.\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C && .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C &&
export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0/bin/:$PATH\" && export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/:$PATH\" &&
export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" && export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" &&
export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
export TASKCLUSTER_NODE_DIR=\"$(cygpath -w $TASKCLUSTER_TASK_DIR/bin)\" && export TASKCLUSTER_NODE_DIR=\"$(cygpath -w $TASKCLUSTER_TASK_DIR/bin)\" &&

6
taskcluster/win-opt-base.tyml
View File

@ -44,7 +44,7 @@ payload:
- .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm"
- echo .\msys64\usr\bin\bash.exe --login -cxe " - echo .\msys64\usr\bin\bash.exe --login -cxe "
export LC_ALL=C && export LC_ALL=C &&
export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:/c/Program Files (x86)/Windows Kits/10/bin/x64/:$PATH\" &&
export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" && export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" &&
export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" &&
(mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR && (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR &&
@ -65,7 +65,7 @@ payload:
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ; $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ;
echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh && echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh &&
cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" | cmd cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" | cmd
/k ""C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"" x64 /k ""C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvarsall.bat"" x64
- .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh &&
exit $TASKCLUSTER_TASK_EXIT_CODE" exit $TASKCLUSTER_TASK_EXIT_CODE"
@ -82,7 +82,7 @@ payload:
- file: pyenv.tar.gz - file: pyenv.tar.gz
content: content:
url: ${system.pyenv.win.url} url: ${system.pyenv.win.url}
- directory: .node-gyp - directory: AppData/Local/node-gyp/Cache
format: tar.gz format: tar.gz
content: content:
url: ${system.node_gyp_cache.url} url: ${system.node_gyp_cache.url}

4
taskcluster/worker.cyml
View File

@ -4,8 +4,8 @@ taskcluster:
provisionerId: proj-deepspeech provisionerId: proj-deepspeech
workerType: ci workerType: ci
workerTypeKvm: kvm workerTypeKvm: kvm
workerTypeWin: win-b workerTypeWin: win
workerTypeCuda: win-gpu workerTypeCuda: win-gpu-b
dockerrpi3: dockerrpi3:
provisionerId: proj-deepspeech provisionerId: proj-deepspeech
workerType: ds-rpi3 workerType: ds-rpi3