diff --git a/.taskcluster.yml b/.taskcluster.yml index 89e4ebbe..9ed80334 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -45,7 +45,7 @@ tasks: - "--login" - "-cxe" - > - echo "deb http://archive.ubuntu.com/ubuntu/ trusty-updates main" > /etc/apt/sources.list.d/trusty-updates.list && + echo "deb http://archive.ubuntu.com/ubuntu/ xenial-updates main" > /etc/apt/sources.list.d/xenial-updates.list && apt-get -qq update && apt-get -qq -y install git python3-pip curl sudo && adduser --system --home /home/build-user build-user && cd /home/build-user/ && diff --git a/Dockerfile.build.tmpl b/Dockerfile.build.tmpl index a4eebb6e..73c8bdd2 100644 --- a/Dockerfile.build.tmpl +++ b/Dockerfile.build.tmpl @@ -1,7 +1,7 @@ # Please refer to the USING documentation, "Dockerfile for building from source" # Need devel version cause we need /usr/include/cudnn.h -FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 +FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 ENV DEEPSPEECH_REPO=#DEEPSPEECH_REPO# ENV DEEPSPEECH_SHA=#DEEPSPEECH_SHA# @@ -46,7 +46,7 @@ RUN update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 1 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 # Install Bazel -RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/0.24.1/bazel_0.24.1-linux-x86_64.deb" +RUN curl -LO "https://github.com/bazelbuild/bazel/releases/download/2.0.0/bazel_2.0.0-linux-x86_64.deb" RUN dpkg -i bazel_*.deb # << END Install base software @@ -56,12 +56,15 @@ RUN dpkg -i bazel_*.deb # Clone TensorFlow from Mozilla repo RUN git clone https://github.com/mozilla/tensorflow/ WORKDIR /tensorflow -RUN git checkout r1.15 +RUN git checkout r2.2 # GPU Environment Setup +ENV TF_NEED_ROCM 0 +ENV TF_NEED_OPENCL_SYCL 0 +ENV TF_NEED_OPENCL 0 ENV TF_NEED_CUDA 1 -ENV TF_CUDA_PATHS "/usr,/usr/local/cuda,/usr/lib/x86_64-linux-gnu/" -ENV TF_CUDA_VERSION 10.0 +ENV TF_CUDA_PATHS "/usr,/usr/local/cuda-10.1,/usr/lib/x86_64-linux-gnu/" +ENV TF_CUDA_VERSION 10.1 ENV TF_CUDNN_VERSION 7.6 ENV TF_CUDA_COMPUTE_CAPABILITIES 6.0 ENV TF_NCCL_VERSION 2.4 @@ -94,7 +97,7 @@ ENV TF_NEED_VERBS 0 ENV TF_NEED_OPENCL_SYCL 0 ENV PYTHON_BIN_PATH /usr/bin/python3.6 -ENV PYTHON_LIB_PATH /usr/lib/python3.6/dist-packages +ENV PYTHON_LIB_PATH /usr/local/lib/python3.6/dist-packages # << END Configure Tensorflow Build diff --git a/doc/USING.rst b/doc/USING.rst index 6496b164..db3013d8 100644 --- a/doc/USING.rst +++ b/doc/USING.rst @@ -28,7 +28,7 @@ Please refer to your system's documentation on how to install these dependencies CUDA dependency ^^^^^^^^^^^^^^^ -The GPU capable builds (Python, NodeJS, C++, etc) depend on the same CUDA runtime as upstream TensorFlow. Currently with TensorFlow 1.15 it depends on CUDA 10.0 and CuDNN v7.6. `See the TensorFlow documentation `_. +The GPU capable builds (Python, NodeJS, C++, etc) depend on the same CUDA runtime as upstream TensorFlow. Currently with TensorFlow 2.2 it depends on CUDA 10.1 and CuDNN v7.6. `See the TensorFlow documentation `_. Getting the pre-trained model ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/native_client/BUILD b/native_client/BUILD index 1e4a66eb..53711dc2 100644 --- a/native_client/BUILD +++ b/native_client/BUILD @@ -1,10 +1,8 @@ # Description: Deepspeech native client library. -load( - "@org_tensorflow//tensorflow:tensorflow.bzl", - "if_cuda", - "tf_cc_shared_object", -) +load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_cc_shared_object") +load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") + load( "@org_tensorflow//tensorflow/lite:build_def.bzl", "tflite_copts", @@ -124,8 +122,8 @@ tf_cc_shared_object( linkopts = select({ "//tensorflow:macos": [], "//tensorflow:linux_x86_64": LINUX_LINKOPTS, - "//tensorflow:rpi3": LINUX_LINKOPTS + ["-l:libstdc++.a"], - "//tensorflow:rpi3-armv8": LINUX_LINKOPTS + ["-l:libstdc++.a"], + "//tensorflow:rpi3": LINUX_LINKOPTS, + "//tensorflow:rpi3-armv8": LINUX_LINKOPTS, "//tensorflow:windows": [], "//conditions:default": [], }) + tflite_linkopts(), @@ -143,7 +141,6 @@ tf_cc_shared_object( ### CPU only build, libdeepspeech.so file size reduced by ~50% "//tensorflow/core/kernels:spectrogram_op", # AudioSpectrogram "//tensorflow/core/kernels:bias_op", # BiasAdd - "//tensorflow/contrib/rnn:lstm_ops_kernels", # BlockLSTM "//tensorflow/core/kernels:cast_op", # Cast "//tensorflow/core/kernels:concat_op", # ConcatV2 "//tensorflow/core/kernels:constant_op", # Const, Placeholder @@ -163,9 +160,10 @@ tf_cc_shared_object( "//tensorflow/core/kernels:softmax_op", # Softmax "//tensorflow/core/kernels:tile_ops", # Tile "//tensorflow/core/kernels:transpose_op", # Transpose + "//tensorflow/core/kernels:rnn_ops", # BlockLSTM # And we also need the op libs for these ops used in the model: "//tensorflow/core:audio_ops_op_lib", # AudioSpectrogram, Mfcc - "//tensorflow/contrib/rnn:lstm_ops_op_lib", # BlockLSTM + "//tensorflow/core:rnn_ops_op_lib", # BlockLSTM "//tensorflow/core:math_ops_op_lib", # Cast, Less, Max, MatMul, Minimum, Range "//tensorflow/core:array_ops_op_lib", # ConcatV2, Const, ExpandDims, Fill, GatherNd, Identity, Pack, Placeholder, Reshape, Tile, Transpose "//tensorflow/core:no_op_op_lib", # NoOp diff --git a/native_client/README.rst b/native_client/README.rst index 841a4546..d77df35d 100644 --- a/native_client/README.rst +++ b/native_client/README.rst @@ -5,8 +5,8 @@ Building DeepSpeech Binaries If you'd like to build the DeepSpeech binaries yourself, you'll need the following pre-requisites downloaded and installed: -* `Mozilla's TensorFlow r1.15 branch `_ -* `Bazel 0.24.1 `_ +* `Mozilla's TensorFlow r2.2 branch `_ +* `Bazel 2.0.0 `_ * `General TensorFlow requirements `_ * `libsox `_ @@ -36,12 +36,12 @@ Clone our fork of TensorFlow and checkout the correct version: .. code-block:: git clone https://github.com/mozilla/tensorflow.git - git checkout origin/r1.15 + git checkout origin/r2.2 Bazel: Download & Install ^^^^^^^^^^^^^^^^^^^^^^^^^ -First, install Bazel 0.24.1 following the `Bazel installation documentation `_. +First, install Bazel 2.0.0 following the `Bazel installation documentation `_. TensorFlow: Configure with Bazel ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/native_client/ctcdecode/Makefile b/native_client/ctcdecode/Makefile index 3cf76478..8bff277b 100644 --- a/native_client/ctcdecode/Makefile +++ b/native_client/ctcdecode/Makefile @@ -45,15 +45,15 @@ workspace_status.cc: # variables over several runs bindings: clean-keep-third-party workspace_status.cc ds-swig pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 - PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) find temp_build -type f -name "*.o" -delete - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build bindings-debug: clean-keep-third-party workspace_status.cc ds-swig pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 - PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 PATH=$(DS_SWIG_BIN_PATH):$(TOOLCHAIN):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py build_ext --debug --num_processes $(NUM_PROCESSES) $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) $(GENERATE_DEBUG_SYMS) find temp_build -type f -name "*.o" -delete - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) LIBEXE=$(LIBEXE) CFLAGS="$(CFLAGS) $(CXXFLAGS) -DDEBUG" LDFLAGS="$(LDFLAGS_NEEDED)" $(PYTHON_PATH) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) rm -rf temp_build diff --git a/native_client/definitions.mk b/native_client/definitions.mk index 41acf5f3..8d43ce0a 100644 --- a/native_client/definitions.mk +++ b/native_client/definitions.mk @@ -48,7 +48,7 @@ endif endif ifeq ($(TARGET),host-win) -TOOLCHAIN := '$(VCINSTALLDIR)\bin\amd64\' +TOOLCHAIN := '$(VCToolsInstallDir)\bin\Hostx64\x64\' TOOL_CC := cl.exe TOOL_CXX := cl.exe TOOL_LD := link.exe @@ -65,7 +65,7 @@ ifeq ($(TARGET),rpi3) TOOLCHAIN ?= ${TFDIR}/bazel-$(shell basename "${TFDIR}")/external/LinaroArmGcc72/bin/arm-linux-gnueabihf- RASPBIAN ?= $(abspath $(NC_DIR)/../multistrap-raspbian-buster) CFLAGS := -march=armv7-a -mtune=cortex-a53 -mfpu=neon-fp-armv8 -mfloat-abi=hard -D_GLIBCXX_USE_CXX11_ABI=0 --sysroot $(RASPBIAN) -CXXFLAGS := $(CXXFLAGS) +CXXFLAGS := $(CFLAGS) LDFLAGS := -Wl,-rpath-link,$(RASPBIAN)/lib/arm-linux-gnueabihf/ -Wl,-rpath-link,$(RASPBIAN)/usr/lib/arm-linux-gnueabihf/ SOX_CFLAGS := -I$(RASPBIAN)/usr/include diff --git a/native_client/dotnet/README.rst b/native_client/dotnet/README.rst index 70fdbf3d..97ac0475 100644 --- a/native_client/dotnet/README.rst +++ b/native_client/dotnet/README.rst @@ -30,11 +30,11 @@ Prerequisites * Windows 10 * `Windows 10 SDK `_ -* `Visual Studio 2017 Community `_ +* `Visual Studio 2019 Community `_ * `Git Large File Storage `_ * `TensorFlow Windows pre-requisites `_ -Inside the Visual Studio Installer enable ``MS Build Tools`` and ``VC++ 2015.3 v14.00 (v140) toolset for desktop``. +Inside the Visual Studio Installer enable ``MS Build Tools`` and ``VC++ 2019 v16.00 (v160) toolset for desktop``. If you want to enable CUDA support you need to follow the steps in `the TensorFlow docs for building on Windows with CUDA `_. @@ -51,7 +51,7 @@ We need to clone ``mozilla/DeepSpeech`` and ``mozilla/tensorflow``. .. code-block:: bash - git clone --branch r1.15 https://github.com/mozilla/tensorflow + git clone --branch r2.2 https://github.com/mozilla/tensorflow Configuring the paths --------------------- @@ -113,7 +113,7 @@ If you run CUDA enabled ``native_client`` we need to add the following to the `` .. code-block:: - C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0\bin + C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.1\bin Building the native_client ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/native_client/javascript/Makefile b/native_client/javascript/Makefile index ac5ee8ed..454bdc42 100644 --- a/native_client/javascript/Makefile +++ b/native_client/javascript/Makefile @@ -41,10 +41,6 @@ package.json: package.json.in package.json.in > package.json && cat package.json npm-dev: package.json -ifeq ($(findstring _NT,$(OS)),_NT) - # node-gyp@5.x behaves erratically with VS2015 and MSBuild.exe detection - $(NPM_TOOL) install node-gyp@4.x -endif $(NPM_TOOL) install --prefix=$(NPM_ROOT)/../ --ignore-scripts --force --verbose --production=false . configure: deepspeech_wrap.cxx package.json npm-dev diff --git a/native_client/javascript/package.json.in b/native_client/javascript/package.json.in index 09dad3a5..42edc3c1 100644 --- a/native_client/javascript/package.json.in +++ b/native_client/javascript/package.json.in @@ -35,12 +35,12 @@ "node-pre-gyp": "0.15.x", "argparse": "1.0.x", "sox-stream": "2.0.x", - "memory-stream": "0.0.3", + "memory-stream": "1.0.x", "node-wav": "0.0.2" }, "devDependencies": { "electron": "^1.7.9", - "node-gyp": "4.x - 5.x", + "node-gyp": "5.x", "typescript": "3.8.x", "typedoc": "0.17.x", "@types/argparse": "1.0.x", diff --git a/native_client/python/Makefile b/native_client/python/Makefile index cf17c9db..7f948649 100644 --- a/native_client/python/Makefile +++ b/native_client/python/Makefile @@ -10,7 +10,7 @@ bindings-clean: # variables over several runs bindings-build: ds-swig pip install --quiet $(PYTHON_PACKAGES) wheel==0.33.6 setuptools==39.1.0 - PATH=$(TOOLCHAIN):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) + DISTUTILS_USE_SDK=1 PATH=$(TOOLCHAIN):$(DS_SWIG_BIN_PATH):$$PATH SWIG_LIB="$(SWIG_LIB)" AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py build_ext $(PYTHON_PLATFORM_NAME) MANIFEST.in: bindings-build > $@ @@ -21,6 +21,6 @@ MANIFEST.in: bindings-build bindings-package: MANIFEST.in cat MANIFEST.in rm -f temp_build/*_wrap.o temp_build/Release/*_wrap.obj - AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) + DISTUTILS_USE_SDK=1 AS=$(AS) CC=$(CC) CXX=$(CXX) LD=$(LD) CFLAGS="$(CFLAGS)" LDFLAGS="$(LDFLAGS_NEEDED) $(RPATH_PYTHON)" MODEL_LDFLAGS="$(LDFLAGS_DIRS)" MODEL_LIBS="$(LIBS)" $(PYTHON_PATH) $(PYTHON_SYSCONFIGDATA) $(NUMPY_INCLUDE) python ./setup.py bdist_wheel $(PYTHON_PLATFORM_NAME) $(SETUP_FLAGS) bindings: bindings-build bindings-package diff --git a/native_client/tfmodelstate.cc b/native_client/tfmodelstate.cc index 5b1e1675..440c44e6 100644 --- a/native_client/tfmodelstate.cc +++ b/native_client/tfmodelstate.cc @@ -118,7 +118,7 @@ TFModelState::init(const char* model_path) int beam_width = metadata_outputs[3].scalar()(); beam_width_ = (unsigned int)(beam_width); - string serialized_alphabet = metadata_outputs[4].scalar()(); + string serialized_alphabet = metadata_outputs[4].scalar()(); err = alphabet_.deserialize(serialized_alphabet.data(), serialized_alphabet.size()); if (err != 0) { return DS_ERR_INVALID_ALPHABET; diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index fee2c1d6..1b94c52d 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -9,7 +9,7 @@ build: dependencies: [] routes: [] maxRunTime: 3600 - docker_image: "ubuntu:14.04" + docker_image: "ubuntu:16.04" system_setup: > true @@ -22,7 +22,7 @@ build: nc_asset_name: 'native_client.tar.xz' args: tests_cmdline: '' - tensorflow_git_desc: 'TensorFlow: v1.15.0-24-gceb46aa' + tensorflow_git_desc: 'TensorFlow: v2.2.0-12-gc29895f' test_model_task: '' homebrew: url: '' diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index 93679741..6a212547 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -1,27 +1,27 @@ python: - packages_trusty: + packages_xenial: apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libpng-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev' packages_buster: apt: 'python3-virtualenv python3-setuptools python3-pip python3-wheel python3-pkg-resources' packages_docs_bionic: apt: 'python3 python3-pip zip doxygen' training: - packages_trusty: + packages_xenial: apt: 'libopus0' tensorflow: - packages_trusty: + packages_xenial: apt: 'make build-essential gfortran git libblas-dev liblapack-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev libpng-dev python zlib1g-dev' java: - packages_trusty: + packages_xenial: apt: 'apt-get -qq -y install curl software-properties-common wget unzip && add-apt-repository --yes ppa:openjdk-r/ppa && apt-get -qq update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y --force-yes install openjdk-8-jdk && java -version && update-ca-certificates -f' electronjs: packages_xenial: apt: 'libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libdbus-1-3 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libx11-xcb1 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 xvfb' nodejs: - packages_trusty: + packages_xenial: apt: 'nodejs sox' apt_pinning: '(echo "Package: nodejs" && echo "Pin: origin deb.nodesource.com" && echo "Pin-Priority: 999") > /etc/apt/preferences' - prep_12: 'echo "deb http://deb.nodesource.com/node_12.x trusty main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' + prep_12: 'echo "deb http://deb.nodesource.com/node_12.x xenial main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' packages_xenial: apt: 'nodejs sox' apt_pinning: '(echo "Package: nodejs" && echo "Pin: origin deb.nodesource.com" && echo "Pin-Priority: 999") > /etc/apt/preferences' @@ -111,14 +111,14 @@ system: namespace: 'project.deepspeech.gradle.6' pyenv: linux: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.7/artifacts/public/pyenv.tar.gz' - namespace: 'project.deepspeech.pyenv.linux.7' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.linux.8/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.linux.8' osx: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.osx.7/artifacts/public/pyenv.tar.gz' - namespace: 'project.deepspeech.pyenv.osx.7' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.osx.8/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.osx.8' win: - url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.7/artifacts/public/pyenv.tar.gz' - namespace: 'project.deepspeech.pyenv.win.7' + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.pyenv.win.8/artifacts/public/pyenv.tar.gz' + namespace: 'project.deepspeech.pyenv.win.8' swig: repo: "https://github.com/lissyx/swig" sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118" @@ -138,4 +138,3 @@ system: osx: '/Users/build-user' win: '/c/builds/tc-workdir' sox_win: '/usr/bin/wget.exe https://sourceforge.net/projects/sox/files/sox/14.4.2/sox-14.4.2-win32.zip/download -O sox-14.4.2-win32.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_TASK_DIR/bin/ -tzip -aoa sox-14.4.2-win32.zip && rm sox-*zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/sox-14.4.2/:$PATH' -aptEc2Mirrors: 'echo "deb http://archive.ubuntu.com/ubuntu/ trusty-updates main" > /etc/apt/sources.list.d/trusty-updates.list && apt-get -qq update && apt-get -qq -y upgrade' diff --git a/taskcluster/android-arm64-cpu-opt.yml b/taskcluster/android-arm64-cpu-opt.yml index fcd1b2be..43b756cc 100644 --- a/taskcluster/android-arm64-cpu-opt.yml +++ b/taskcluster/android-arm64-cpu-opt.yml @@ -8,7 +8,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-arm64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-arm64" - "index.project.deepspeech.deepspeech.native_client.android-arm64.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.android-arm64/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-arm64/artifacts/public/home.tar.xz" scripts: build: "taskcluster/android-build.sh arm64-v8a" package: "taskcluster/android-package.sh arm64-v8a" diff --git a/taskcluster/android-armv7-cpu-opt.yml b/taskcluster/android-armv7-cpu-opt.yml index 2578758b..168b542b 100644 --- a/taskcluster/android-armv7-cpu-opt.yml +++ b/taskcluster/android-armv7-cpu-opt.yml @@ -8,7 +8,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-armv7" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-armv7" - "index.project.deepspeech.deepspeech.native_client.android-armv7.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.android-armv7/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-armv7/artifacts/public/home.tar.xz" scripts: build: "taskcluster/android-build.sh armeabi-v7a" package: "taskcluster/android-package.sh armeabi-v7a" diff --git a/taskcluster/android-cache-arm64-v8a-android-24.yml b/taskcluster/android-cache-arm64-v8a-android-24.yml index 8fa8e927..260ea546 100644 --- a/taskcluster/android-cache-arm64-v8a-android-24.yml +++ b/taskcluster/android-cache-arm64-v8a-android-24.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.arm64_v8a.android_24.url} artifact_namespace: ${system.android_cache.arm64_v8a.android_24.namespace} diff --git a/taskcluster/android-cache-arm64-v8a-android-25.yml b/taskcluster/android-cache-arm64-v8a-android-25.yml index 36638726..34b8158b 100644 --- a/taskcluster/android-cache-arm64-v8a-android-25.yml +++ b/taskcluster/android-cache-arm64-v8a-android-25.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.arm64_v8a.android_25.url} artifact_namespace: ${system.android_cache.arm64_v8a.android_25.namespace} diff --git a/taskcluster/android-cache-armeabi-v7a-android-24.yml b/taskcluster/android-cache-armeabi-v7a-android-24.yml index 4c686739..441b9255 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-24.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-24.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.armeabi_v7a.android_24.url} artifact_namespace: ${system.android_cache.armeabi_v7a.android_24.namespace} diff --git a/taskcluster/android-cache-armeabi-v7a-android-25.yml b/taskcluster/android-cache-armeabi-v7a-android-25.yml index e4f78de7..620745a0 100644 --- a/taskcluster/android-cache-armeabi-v7a-android-25.yml +++ b/taskcluster/android-cache-armeabi-v7a-android-25.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.armeabi_v7a.android_25.url} artifact_namespace: ${system.android_cache.armeabi_v7a.android_25.namespace} diff --git a/taskcluster/android-cache-sdk-android-27.yml b/taskcluster/android-cache-sdk-android-27.yml index 668e142e..0b047dc1 100644 --- a/taskcluster/android-cache-sdk-android-27.yml +++ b/taskcluster/android-cache-sdk-android-27.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.sdk.android_27.url} artifact_namespace: ${system.android_cache.sdk.android_27.namespace} diff --git a/taskcluster/android-cache-x86_64-android-24.yml b/taskcluster/android-cache-x86_64-android-24.yml index ff06f551..c3b3ba1d 100644 --- a/taskcluster/android-cache-x86_64-android-24.yml +++ b/taskcluster/android-cache-x86_64-android-24.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.x86_64.android_24.url} artifact_namespace: ${system.android_cache.x86_64.android_24.namespace} diff --git a/taskcluster/android-cache-x86_64-android-25.yml b/taskcluster/android-cache-x86_64-android-25.yml index 8ccac4f9..a2e642e6 100644 --- a/taskcluster/android-cache-x86_64-android-25.yml +++ b/taskcluster/android-cache-x86_64-android-25.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.x86_64.android_25.url} artifact_namespace: ${system.android_cache.x86_64.android_25.namespace} diff --git a/taskcluster/android-cache-x86_64-android-26.yml b/taskcluster/android-cache-x86_64-android-26.yml index 0a74e78e..a8b9b15a 100644 --- a/taskcluster/android-cache-x86_64-android-26.yml +++ b/taskcluster/android-cache-x86_64-android-26.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.x86_64.android_26.url} artifact_namespace: ${system.android_cache.x86_64.android_26.namespace} diff --git a/taskcluster/android-cache-x86_64-android-28.yml b/taskcluster/android-cache-x86_64-android-28.yml index 4fa124fd..6193dd00 100644 --- a/taskcluster/android-cache-x86_64-android-28.yml +++ b/taskcluster/android-cache-x86_64-android-28.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.x86_64.android_28.url} artifact_namespace: ${system.android_cache.x86_64.android_28.namespace} diff --git a/taskcluster/android-cache-x86_64-android-29.yml b/taskcluster/android-cache-x86_64-android-29.yml index 57c194f2..63e1b3c6 100644 --- a/taskcluster/android-cache-x86_64-android-29.yml +++ b/taskcluster/android-cache-x86_64-android-29.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.x86_64.android_29.url} artifact_namespace: ${system.android_cache.x86_64.android_29.namespace} diff --git a/taskcluster/android-cache-x86_64-android-30.yml b/taskcluster/android-cache-x86_64-android-30.yml index 217459aa..51cf50b9 100644 --- a/taskcluster/android-cache-x86_64-android-30.yml +++ b/taskcluster/android-cache-x86_64-android-30.yml @@ -2,7 +2,7 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} cache: artifact_url: ${system.android_cache.x86_64.android_30.url} artifact_namespace: ${system.android_cache.x86_64.android_30.namespace} diff --git a/taskcluster/android-java-opt.yml b/taskcluster/android-java-opt.yml index 7ca7d0f2..5d4ec601 100644 --- a/taskcluster/android-java-opt.yml +++ b/taskcluster/android-java-opt.yml @@ -13,8 +13,8 @@ build: - "index.project.deepspeech.deepspeech.native_client.android-apk.${event.head.sha}" system_setup: > - ${java.packages_trusty.apt} - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.android-armv7/artifacts/public/home.tar.xz" + ${java.packages_xenial.apt} + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-armv7/artifacts/public/home.tar.xz" gradle_cache: url: ${system.gradle_cache.url} namespace: ${system.gradle_cache.namespace} diff --git a/taskcluster/android-x86_64-cpu-opt.yml b/taskcluster/android-x86_64-cpu-opt.yml index 447822ab..a304fe9f 100644 --- a/taskcluster/android-x86_64-cpu-opt.yml +++ b/taskcluster/android-x86_64-cpu-opt.yml @@ -8,7 +8,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.android-x86_64" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.android-x86_64" - "index.project.deepspeech.deepspeech.native_client.android-x86_64.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.android-arm64/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.android-arm64/artifacts/public/home.tar.xz" scripts: build: "taskcluster/android-build.sh x86_64" package: "taskcluster/android-package.sh x86_64" diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index 85894360..12f41b2f 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -9,7 +9,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx" - "index.project.deepspeech.deepspeech.native_client.osx.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.osx/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/darwin-amd64-ctc-opt.yml b/taskcluster/darwin-amd64-ctc-opt.yml index 0e3e51e6..0f80f31e 100644 --- a/taskcluster/darwin-amd64-ctc-opt.yml +++ b/taskcluster/darwin-amd64-ctc-opt.yml @@ -9,7 +9,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc" - "index.project.deepspeech.deepspeech.native_client.osx-ctc.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.osx/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz" maxRunTime: 14400 scripts: build: 'taskcluster/decoder-build.sh' diff --git a/taskcluster/darwin-amd64-tflite-opt.yml b/taskcluster/darwin-amd64-tflite-opt.yml index 4ab6c7c7..4a22e0dc 100644 --- a/taskcluster/darwin-amd64-tflite-opt.yml +++ b/taskcluster/darwin-amd64-tflite-opt.yml @@ -9,7 +9,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite" - "index.project.deepspeech.deepspeech.native_client.osx-tflite.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.osx/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.osx/artifacts/public/home.tar.xz" scripts: build: "taskcluster/host-build.sh tflite" package: "taskcluster/package.sh" diff --git a/taskcluster/docker-build-base.tyml b/taskcluster/docker-build-base.tyml index 6e4d2471..6d46a4bd 100644 --- a/taskcluster/docker-build-base.tyml +++ b/taskcluster/docker-build-base.tyml @@ -30,7 +30,6 @@ then: dockerfile: { $eval: strip(str(build.dockerfile)) } in: > apt-get -qq -y remove --purge ubuntu-advantage-tools && - ${aptEc2Mirrors} && apt-get -qq update && apt-get -qq -y install git wget pkg-config apt-transport-https ca-certificates curl software-properties-common make && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && diff --git a/taskcluster/generic_tc_caching-linux-opt-base.tyml b/taskcluster/generic_tc_caching-linux-opt-base.tyml index 36668a0c..da91f115 100644 --- a/taskcluster/generic_tc_caching-linux-opt-base.tyml +++ b/taskcluster/generic_tc_caching-linux-opt-base.tyml @@ -25,7 +25,7 @@ payload: taskIndexExpire: { $fromNow: '6 months' } in: > (apt-get -qq -y remove --purge ubuntu-advantage-tools || true) && - apt-get -qq update && apt-get -qq -y install curl git && ${extraSystemSetup}; + apt-get -qq update && apt-get -qq -y install curl git sudo && ${extraSystemSetup}; cache_artifact=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.cache.artifact_url}` && if [ "$cache_artifact" != "200" ]; then adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux}/ && diff --git a/taskcluster/gradle-cache.yml b/taskcluster/gradle-cache.yml index ef589197..a77b2e19 100644 --- a/taskcluster/gradle-cache.yml +++ b/taskcluster/gradle-cache.yml @@ -5,7 +5,7 @@ build: artifact_namespace: ${system.gradle_cache.namespace} system_setup: > - ${java.packages_trusty.apt} + ${java.packages_xenial.apt} scripts: build: "taskcluster/gradle-build.sh" package: "taskcluster/gradle-package.sh" diff --git a/taskcluster/linux-amd64-cpu-opt.yml b/taskcluster/linux-amd64-cpu-opt.yml index e2af482b..50f78a2d 100644 --- a/taskcluster/linux-amd64-cpu-opt.yml +++ b/taskcluster/linux-amd64-cpu-opt.yml @@ -10,9 +10,9 @@ build: - "index.project.deepspeech.deepspeech.native_client.cpu.${event.head.sha}" system_setup: > - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz" scripts: build: "taskcluster/host-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-amd64-ctc-opt.yml b/taskcluster/linux-amd64-ctc-opt.yml index 00af6149..f0979261 100644 --- a/taskcluster/linux-amd64-ctc-opt.yml +++ b/taskcluster/linux-amd64-ctc-opt.yml @@ -10,9 +10,9 @@ build: - "index.project.deepspeech.deepspeech.native_client.cpu-ctc.${event.head.sha}" system_setup: > - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz" scripts: build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' diff --git a/taskcluster/linux-amd64-gpu-opt.yml b/taskcluster/linux-amd64-gpu-opt.yml index 532b8bb7..f34f2de0 100644 --- a/taskcluster/linux-amd64-gpu-opt.yml +++ b/taskcluster/linux-amd64-gpu-opt.yml @@ -10,9 +10,9 @@ build: - "index.project.deepspeech.deepspeech.native_client.gpu.${event.head.sha}" system_setup: > - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.gpu/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.gpu/artifacts/public/home.tar.xz" maxRunTime: 14400 scripts: build: "taskcluster/cuda-build.sh" diff --git a/taskcluster/linux-amd64-tflite-opt.yml b/taskcluster/linux-amd64-tflite-opt.yml index 3d4847f9..ffdb7499 100644 --- a/taskcluster/linux-amd64-tflite-opt.yml +++ b/taskcluster/linux-amd64-tflite-opt.yml @@ -10,9 +10,9 @@ build: - "index.project.deepspeech.deepspeech.native_client.tflite.${event.head.sha}" system_setup: > - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.cpu/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.cpu/artifacts/public/home.tar.xz" scripts: build: "taskcluster/host-build.sh tflite" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-arm64-cpu-opt.yml b/taskcluster/linux-arm64-cpu-opt.yml index 1865784e..aa4e2a34 100644 --- a/taskcluster/linux-arm64-cpu-opt.yml +++ b/taskcluster/linux-arm64-cpu-opt.yml @@ -14,12 +14,12 @@ build: apt-get -qq -y install gdebi git pixz && wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb && - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml system_config: > multistrap -d /tmp/multistrap-armbian64-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_armbian64_buster.conf - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm64/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.arm64/artifacts/public/home.tar.xz" scripts: build: "taskcluster/arm64-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/linux-opt-base.tyml b/taskcluster/linux-opt-base.tyml index 0298954c..f98d84a6 100644 --- a/taskcluster/linux-opt-base.tyml +++ b/taskcluster/linux-opt-base.tyml @@ -30,7 +30,7 @@ then: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "ubuntu:14.04" + image: "ubuntu:16.04" env: TENSORFLOW_BUILD_ARTIFACT: ${build.tensorflow} @@ -43,10 +43,8 @@ then: extraSystemSetup: { $eval: strip(str(build.system_setup)) } extraSystemConfig: { $eval: strip(str(build.system_config)) } in: > - apt-get -qq -y remove --purge ubuntu-advantage-tools && - ${aptEc2Mirrors} && adduser --system --home ${system.homedir.linux} ${system.username} && - apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_trusty.apt} pixz pkg-config realpath unzip wget zip && ${extraSystemSetup} && + apt-get -qq update && apt-get -qq -y install ${tensorflow.packages_xenial.apt} pixz pkg-config realpath sudo unzip wget zip && ${extraSystemSetup} && cd ${system.homedir.linux}/ && echo -e "#!/bin/bash\nset -xe\n env && id && (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | tar -C ${system.homedir.linux}/ -xf - ) && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s ~/DeepSpeech/ds/native_client/ ~/DeepSpeech/tf/native_client && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf - && mkdir -p ${system.homedir.linux}/pyenv-root/ && wget -O - ${system.pyenv.linux.url} | tar -C ${system.homedir.linux}/pyenv-root/ -xzf - && if [ ! -z "${build.gradle_cache.url}" ]; then wget -O - ${build.gradle_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi && if [ ! -z "${build.android_cache.url}" ]; then wget -O - ${build.android_cache.url} | tar -C ${system.homedir.linux}/ -xzf - ; fi;" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && diff --git a/taskcluster/linux-rpi3-cpu-opt.yml b/taskcluster/linux-rpi3-cpu-opt.yml index 0cabb870..82959259 100644 --- a/taskcluster/linux-rpi3-cpu-opt.yml +++ b/taskcluster/linux-rpi3-cpu-opt.yml @@ -14,12 +14,12 @@ build: apt-get -qq -y install gdebi git pixz && wget http://mirrors.kernel.org/ubuntu/pool/universe/m/multistrap/multistrap_2.2.0ubuntu2_all.deb -O /tmp/multistrap_2.2.0ubuntu2_all.deb && echo "y" | gdebi /tmp/multistrap_2.2.0ubuntu2_all.deb && - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml system_config: > multistrap -d /tmp/multistrap-raspbian-buster/ -f ${system.homedir.linux}/DeepSpeech/ds/native_client/multistrap_raspbian_buster.conf - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.arm/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.arm/artifacts/public/home.tar.xz" scripts: build: "taskcluster/rpi3-build.sh" package: "taskcluster/package.sh" diff --git a/taskcluster/node-package-cpu.yml b/taskcluster/node-package-cpu.yml index 72870cd5..6769543b 100644 --- a/taskcluster/node-package-cpu.yml +++ b/taskcluster/node-package-cpu.yml @@ -8,7 +8,7 @@ build: - "win-amd64-cpu-opt" system_setup: > - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml scripts: build: "taskcluster/node-build.sh" diff --git a/taskcluster/node-package-gpu.yml b/taskcluster/node-package-gpu.yml index db48ae32..a430b4b1 100644 --- a/taskcluster/node-package-gpu.yml +++ b/taskcluster/node-package-gpu.yml @@ -5,7 +5,7 @@ build: - "win-amd64-gpu-opt" system_setup: > - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml scripts: build: "taskcluster/node-build.sh --cuda" diff --git a/taskcluster/node-package-opt-base.tyml b/taskcluster/node-package-opt-base.tyml index b17daaac..b14cdd4c 100644 --- a/taskcluster/node-package-opt-base.tyml +++ b/taskcluster/node-package-opt-base.tyml @@ -30,7 +30,7 @@ then: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "ubuntu:14.04" + image: "ubuntu:16.04" command: - "/bin/bash" @@ -40,10 +40,8 @@ then: extraSystemSetup: { $eval: strip(str(build.system_setup)) } extraSystemConfig: { $eval: strip(str(build.system_config)) } in: > - apt-get -qq -y remove --purge ubuntu-advantage-tools && - ${aptEc2Mirrors} && adduser --system --home ${system.homedir.linux} ${system.username} && - apt-get -qq update && apt-get -qq -y install realpath git wget curl make && ${extraSystemSetup} && + apt-get -qq update && apt-get -qq -y install realpath git wget curl make sudo && ${extraSystemSetup} && cd ${system.homedir.linux}/ && echo -e "#!/bin/bash\nset -xe\n env && id && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && mkdir -p ~/DeepSpeech/tf/ && touch ~/DeepSpeech/tf/tc-vars.sh && chmod +x ~/DeepSpeech/tf/tc-vars.sh && mkdir -p ${system.homedir.linux}/.cache/node-gyp/ && wget -O - ${system.node_gyp_cache.url} | tar -C ${system.homedir.linux}/.cache/node-gyp/ -xzf -" > /tmp/clone.sh && chmod +x /tmp/clone.sh && sudo -H -u ${system.username} /bin/bash /tmp/clone.sh && ${extraSystemConfig} && diff --git a/taskcluster/node-package-tflite.yml b/taskcluster/node-package-tflite.yml index d49404d0..00338446 100644 --- a/taskcluster/node-package-tflite.yml +++ b/taskcluster/node-package-tflite.yml @@ -6,7 +6,7 @@ build: - "win-amd64-tflite-opt" system_setup: > - ${nodejs.packages_trusty.prep_12} && ${nodejs.packages_trusty.apt_pinning} + ${nodejs.packages_xenial.prep_12} && ${nodejs.packages_xenial.apt_pinning} && apt-get -qq update && apt-get -qq -y install nodejs python-yaml scripts: build: "taskcluster/node-build.sh --tflite" diff --git a/taskcluster/pyenv-build.sh b/taskcluster/pyenv-build.sh index b3faf8fb..54d11c7a 100755 --- a/taskcluster/pyenv-build.sh +++ b/taskcluster/pyenv-build.sh @@ -18,11 +18,7 @@ for pyver_conf in ${SUPPORTED_PYTHON_VERSIONS}; do pyalias="${pyver}_${pyconf}" - maybe_ssl102_py37 ${pyver} - - LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ - PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \ - pyenv_install ${pyver} ${pyalias} + PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf}" pyenv_install ${pyver} ${pyalias} setup_pyenv_virtualenv "${pyalias}" "deepspeech" diff --git a/taskcluster/pyenv-linux-amd64.yml b/taskcluster/pyenv-linux-amd64.yml index a8ece0b4..4946e15b 100644 --- a/taskcluster/pyenv-linux-amd64.yml +++ b/taskcluster/pyenv-linux-amd64.yml @@ -5,7 +5,7 @@ build: artifact_namespace: "${system.pyenv.linux.namespace}" system_setup: > - apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_trusty.apt} wget + apt-get -qq update && apt-get -qq -y install python-yaml ${python.packages_xenial.apt} wget scripts: build: "taskcluster/pyenv-build.sh" package: "taskcluster/pyenv-package.sh" diff --git a/taskcluster/swig-linux-amd64.yml b/taskcluster/swig-linux-amd64.yml index 3b199c1a..2a127bbc 100644 --- a/taskcluster/swig-linux-amd64.yml +++ b/taskcluster/swig-linux-amd64.yml @@ -1,6 +1,6 @@ build: template_file: generic_tc_caching-linux-opt-base.tyml - docker_image: "ubuntu:14.04" + docker_image: "ubuntu:16.04" build_or_cache: repo: "${system.swig.repo}" sha: "${system.swig.sha1}" diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 1a38e18b..2efc1dbc 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -42,10 +42,6 @@ if [ "${OS}" = "Darwin" ]; then fi; fi; -PY37_OPENSSL_DIR="${PYENV_ROOT}/ssl-xenial" -export PY37_LDPATH="${PY37_OPENSSL_DIR}/usr/lib/" -export LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH - export TASKCLUSTER_ARTIFACTS=${TASKCLUSTER_ARTIFACTS:-/tmp/artifacts} export TASKCLUSTER_TMP_DIR=${TASKCLUSTER_TMP_DIR:-/tmp} diff --git a/taskcluster/tc-build-utils.sh b/taskcluster/tc-build-utils.sh index 5530517e..6a41a88a 100755 --- a/taskcluster/tc-build-utils.sh +++ b/taskcluster/tc-build-utils.sh @@ -231,7 +231,7 @@ do_deepspeech_netframework_build() # Setup dependencies nuget install DeepSpeechConsole/packages.config -OutputDirectory packages/ - MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin\MSBuild.exe')" + MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin\MSBuild.exe')" # We need MSYS2_ARG_CONV_EXCL='/' otherwise the '/' of CLI parameters gets mangled and disappears # We build the .NET Client for .NET Framework v4.5,v4.6,v4.7 @@ -270,7 +270,7 @@ do_deepspeech_netframework_wpf_example_build() # Setup dependencies nuget install DeepSpeechWPF/packages.config -OutputDirectory DeepSpeechWPF/packages/ - MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin\MSBuild.exe')" + MSBUILD="$(cygpath 'C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin\MSBuild.exe')" # We need MSYS2_ARG_CONV_EXCL='/' otherwise the '/' of CLI parameters gets mangled and disappears # Build WPF example diff --git a/taskcluster/tc-py-utils.sh b/taskcluster/tc-py-utils.sh index 2beee417..36f80731 100755 --- a/taskcluster/tc-py-utils.sh +++ b/taskcluster/tc-py-utils.sh @@ -191,47 +191,6 @@ pyenv_install() fi } -# Hack to extract Ubuntu's 16.04 libssl 1.0.2 packages and use them during the -# local build of Python. -# -# Avoid (risky) upgrade of base system, allowing to keep one task build that -# builds all the python packages -maybe_ssl102_py37() -{ - pyver=$1 - - unset PY37_OPENSSL - - ARCH=$(uname -m) - case "${pyver}" in - 3.7*|3.8*) - if [ "${OS}" = "Linux" -a "${ARCH}" = "x86_64" ]; then - if [ -d "${PY37_OPENSSL_DIR}" ]; then - rm -rf "${PY37_OPENSSL_DIR}" - fi - - mkdir -p ${PY37_OPENSSL_DIR} - ${WGET} -P ${TASKCLUSTER_TMP_DIR} \ - http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl-dev_1.0.2g-1ubuntu4.16_amd64.deb \ - http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.0.0_1.0.2g-1ubuntu4.16_amd64.deb - - for deb in ${TASKCLUSTER_TMP_DIR}/libssl*.deb; do - dpkg -x ${deb} ${PY37_OPENSSL_DIR} - done; - - # Python configure expects things to be under lib/ - mv ${PY37_OPENSSL_DIR}/usr/include/x86_64-linux-gnu/openssl/opensslconf.h ${PY37_OPENSSL_DIR}/usr/include/openssl/ - mv ${PY37_OPENSSL_DIR}/lib/x86_64-linux-gnu/lib* ${PY37_OPENSSL_DIR}/usr/lib/ - mv ${PY37_OPENSSL_DIR}/usr/lib/x86_64-linux-gnu/* ${PY37_OPENSSL_DIR}/usr/lib/ - ln -sfn libcrypto.so.1.0.0 ${PY37_OPENSSL_DIR}/usr/lib/libcrypto.so - ln -sfn libssl.so.1.0.0 ${PY37_OPENSSL_DIR}/usr/lib/libssl.so - - export PY37_OPENSSL="--with-openssl=${PY37_OPENSSL_DIR}/usr" - fi; - ;; - esac -} - maybe_numpy_min_version() { local pyver=$1 diff --git a/taskcluster/test-armbian-opt-base.tyml b/taskcluster/test-armbian-opt-base.tyml index 7218e684..afe7b64b 100644 --- a/taskcluster/test-armbian-opt-base.tyml +++ b/taskcluster/test-armbian-opt-base.tyml @@ -19,7 +19,7 @@ then: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "arm64v8/debian:buster-20190812" + image: "arm64v8/debian:buster-20200422" env: $let: @@ -42,7 +42,7 @@ then: - $let: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > - apt-get -qq update && apt-get -qq -y install curl python-simplejson git gnupg pixz sox sudo wget libatlas3-base libopenblas-base xxd && ${extraSystemSetup} && + apt-get -qq update && apt-get -qq -y install curl python-simplejson git gnupg pixz sox sudo wget libatlas3-base libopenblas-base xxd && update-ca-certificates --fresh && ${extraSystemSetup} && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux} && echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && diff --git a/taskcluster/test-augmentations-linux-amd64-py36m-opt.yml b/taskcluster/test-augmentations-linux-amd64-py36m-opt.yml index 5dd066c2..8007cd6c 100644 --- a/taskcluster/test-augmentations-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-augmentations-linux-amd64-py36m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-augmentation-tests.sh 3.6.10:m" metadata: diff --git a/taskcluster/test-linux-opt-base.tyml b/taskcluster/test-linux-opt-base.tyml index fcb8b466..67f8220b 100644 --- a/taskcluster/test-linux-opt-base.tyml +++ b/taskcluster/test-linux-opt-base.tyml @@ -44,7 +44,6 @@ then: - $let: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > - ${aptEc2Mirrors} && apt-get -qq update && apt-get -qq -y install curl python-simplejson git pixz sox sudo wget && ${extraSystemSetup} && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux} && diff --git a/taskcluster/test-linux-opt-tag-base.tyml b/taskcluster/test-linux-opt-tag-base.tyml index 1e6686b5..78c87625 100644 --- a/taskcluster/test-linux-opt-tag-base.tyml +++ b/taskcluster/test-linux-opt-tag-base.tyml @@ -44,7 +44,6 @@ then: - $let: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > - ${aptEc2Mirrors} && apt-get -qq update && apt-get -qq -y install curl python-simplejson git pixz sox sudo wget && ${extraSystemSetup} && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux} && diff --git a/taskcluster/test-raspbian-opt-base.tyml b/taskcluster/test-raspbian-opt-base.tyml index f3d2a5e5..2938b6b6 100644 --- a/taskcluster/test-raspbian-opt-base.tyml +++ b/taskcluster/test-raspbian-opt-base.tyml @@ -19,7 +19,7 @@ then: payload: maxRunTime: { $eval: to_int(build.maxRunTime) } - image: "resin/rpi-raspbian:buster-20190121" + image: "balenalib/rpi-raspbian:buster-20200429" env: $let: @@ -42,7 +42,7 @@ then: - $let: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > - apt-get -qq update && apt-get -qq -y install curl python-simplejson git pixz sox wget libatlas3-base xxd && ${extraSystemSetup} && + apt-get -qq update && apt-get -qq -y install curl python-simplejson git pixz sox wget libatlas3-base xxd && update-ca-certificates --fresh && ${extraSystemSetup} && adduser --system --home ${system.homedir.linux} ${system.username} && cd ${system.homedir.linux} && echo -e "#!/bin/bash\nset -xe\n env && id && mkdir ~/DeepSpeech/ && git clone --quiet ${event.head.repo.url} ~/DeepSpeech/ds/ && cd ~/DeepSpeech/ds && git checkout --quiet ${event.head.sha}" > /tmp/clone.sh && chmod +x /tmp/clone.sh && diff --git a/taskcluster/test-training-extra_16k-linux-amd64-py35m-opt.yml b/taskcluster/test-training-extra_16k-linux-amd64-py35m-opt.yml index 6d19f491..3b1e1378 100644 --- a/taskcluster/test-training-extra_16k-linux-amd64-py35m-opt.yml +++ b/taskcluster/test-training-extra_16k-linux-amd64-py35m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-extra-tests.sh 3.5.8:m 16k" metadata: diff --git a/taskcluster/test-training-extra_16k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-extra_16k-linux-amd64-py36m-opt.yml index e292a841..c75bc7a2 100644 --- a/taskcluster/test-training-extra_16k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-extra_16k-linux-amd64-py36m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-extra-tests.sh 3.6.10:m 16k" metadata: diff --git a/taskcluster/test-training-extra_16k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-extra_16k-linux-amd64-py37m-opt.yml index a41f7cd4..79f9419d 100644 --- a/taskcluster/test-training-extra_16k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training-extra_16k-linux-amd64-py37m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-extra-tests.sh 3.7.6:m 16k" metadata: diff --git a/taskcluster/test-training-extra_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-extra_8k-linux-amd64-py36m-opt.yml index b79209ae..a5335c50 100644 --- a/taskcluster/test-training-extra_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-extra_8k-linux-amd64-py36m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-extra-tests.sh 3.6.10:m 8k" metadata: diff --git a/taskcluster/test-training-extra_8k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-extra_8k-linux-amd64-py37m-opt.yml index ed5259bb..32edde28 100644 --- a/taskcluster/test-training-extra_8k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training-extra_8k-linux-amd64-py37m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-extra-tests.sh 3.7.6:m 8k" metadata: diff --git a/taskcluster/test-training-pypi_16k-linux-amd64-py35m-opt.yml b/taskcluster/test-training-pypi_16k-linux-amd64-py35m-opt.yml index dc7f83d8..67a14191 100644 --- a/taskcluster/test-training-pypi_16k-linux-amd64-py35m-opt.yml +++ b/taskcluster/test-training-pypi_16k-linux-amd64-py35m-opt.yml @@ -7,7 +7,7 @@ build: ref_match: "refs/tags/" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.5.8:m 16k --pypi" metadata: diff --git a/taskcluster/test-training-pypi_16k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-pypi_16k-linux-amd64-py36m-opt.yml index ffac9c89..f2909aa1 100644 --- a/taskcluster/test-training-pypi_16k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-pypi_16k-linux-amd64-py36m-opt.yml @@ -7,7 +7,7 @@ build: ref_match: "refs/tags/" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 16k --pypi" metadata: diff --git a/taskcluster/test-training-pypi_16k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-pypi_16k-linux-amd64-py37m-opt.yml index a7624e73..c3b8282a 100644 --- a/taskcluster/test-training-pypi_16k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training-pypi_16k-linux-amd64-py37m-opt.yml @@ -7,7 +7,7 @@ build: ref_match: "refs/tags/" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.7.6:m 16k --pypi" metadata: diff --git a/taskcluster/test-training-pypi_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-pypi_8k-linux-amd64-py36m-opt.yml index 48c52ea1..78029d23 100644 --- a/taskcluster/test-training-pypi_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-pypi_8k-linux-amd64-py36m-opt.yml @@ -7,7 +7,7 @@ build: ref_match: "refs/tags/" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 8k --pypi" metadata: diff --git a/taskcluster/test-training-pypi_8k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-pypi_8k-linux-amd64-py37m-opt.yml index 18f4d51f..c044a550 100644 --- a/taskcluster/test-training-pypi_8k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training-pypi_8k-linux-amd64-py37m-opt.yml @@ -7,7 +7,7 @@ build: ref_match: "refs/tags/" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.7.6:m 8k --pypi" metadata: diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml index d33ad21d..3bd4c39d 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py35m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.5.8:m" metadata: diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml index 195ddfe9..a4949830 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py36m-opt.yml @@ -4,10 +4,10 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.6.10:m" metadata: name: "DeepSpeech on Linux AMD64 CPU training unittests using Python 3.6" description: "Training unittests DeepSpeech LDC93S1 model for Linux/AMD64 using Python 3.6, for CPU only, and optimized version" - \ No newline at end of file + diff --git a/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml index 1f9aabc3..5b9ff7a5 100644 --- a/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training-unittests_8k-linux-amd64-py37m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-unittests.sh 3.7.6:m" metadata: diff --git a/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml b/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml index 48c25019..5e5e2f0a 100644 --- a/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml +++ b/taskcluster/test-training_16k-linux-amd64-py35m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.5.8:m 16k" metadata: diff --git a/taskcluster/test-training_16k-linux-amd64-py36m-opt.yml b/taskcluster/test-training_16k-linux-amd64-py36m-opt.yml index 1d4eaa98..7f19771d 100644 --- a/taskcluster/test-training_16k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training_16k-linux-amd64-py36m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 16k" metadata: diff --git a/taskcluster/test-training_16k-linux-amd64-py37m-opt.yml b/taskcluster/test-training_16k-linux-amd64-py37m-opt.yml index 25a35efa..4146dfca 100644 --- a/taskcluster/test-training_16k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training_16k-linux-amd64-py37m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.7.6:m 16k" metadata: diff --git a/taskcluster/test-training_8k-linux-amd64-py36m-opt.yml b/taskcluster/test-training_8k-linux-amd64-py36m-opt.yml index b9396fdd..560ffcfb 100644 --- a/taskcluster/test-training_8k-linux-amd64-py36m-opt.yml +++ b/taskcluster/test-training_8k-linux-amd64-py36m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.6.10:m 8k" metadata: diff --git a/taskcluster/test-training_8k-linux-amd64-py37m-opt.yml b/taskcluster/test-training_8k-linux-amd64-py37m-opt.yml index eb7cb1be..09bfd129 100644 --- a/taskcluster/test-training_8k-linux-amd64-py37m-opt.yml +++ b/taskcluster/test-training_8k-linux-amd64-py37m-opt.yml @@ -4,7 +4,7 @@ build: - "linux-amd64-ctc-opt" system_setup: > - apt-get -qq update && apt-get -qq -y install ${training.packages_trusty.apt} + apt-get -qq update && apt-get -qq -y install ${training.packages_xenial.apt} args: tests_cmdline: "${system.homedir.linux}/DeepSpeech/ds/taskcluster/tc-train-tests.sh 3.7.6:m 8k" metadata: diff --git a/taskcluster/test-win-cuda-opt-base.tyml b/taskcluster/test-win-cuda-opt-base.tyml index eb2fb1d1..e5dc5ee6 100644 --- a/taskcluster/test-win-cuda-opt-base.tyml +++ b/taskcluster/test-win-cuda-opt-base.tyml @@ -44,7 +44,7 @@ then: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > .\msys64\usr\bin\bash.exe --login -cxe "export LC_ALL=C && - export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0/bin/:$PATH\" && + export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/:$PATH\" && export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" && export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && export TASKCLUSTER_NODE_DIR=\"$(cygpath -w $TASKCLUSTER_TASK_DIR/bin)\" && diff --git a/taskcluster/win-amd64-cpu-opt.yml b/taskcluster/win-amd64-cpu-opt.yml index 32512094..b7dbba5b 100644 --- a/taskcluster/win-amd64-cpu-opt.yml +++ b/taskcluster/win-amd64-cpu-opt.yml @@ -8,7 +8,7 @@ build: - "node-gyp-cache" - "swig-win-amd64" - "pyenv-win-amd64" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh" package: "taskcluster/win-package.sh" diff --git a/taskcluster/win-amd64-ctc-opt.yml b/taskcluster/win-amd64-ctc-opt.yml index ebd37445..b17bc53c 100644 --- a/taskcluster/win-amd64-ctc-opt.yml +++ b/taskcluster/win-amd64-ctc-opt.yml @@ -8,7 +8,7 @@ build: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.win-ctc" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.win-ctc" - "index.project.deepspeech.deepspeech.native_client.win-ctc.${event.head.sha}" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz" scripts: build: 'taskcluster/decoder-build.sh' package: 'taskcluster/decoder-package.sh' diff --git a/taskcluster/win-amd64-gpu-opt.yml b/taskcluster/win-amd64-gpu-opt.yml index 077e79db..b9c99395 100644 --- a/taskcluster/win-amd64-gpu-opt.yml +++ b/taskcluster/win-amd64-gpu-opt.yml @@ -8,7 +8,7 @@ build: - "node-gyp-cache" - "swig-win-amd64" - "pyenv-win-amd64" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win-cuda/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win-cuda/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh --cuda" package: "taskcluster/win-package.sh" diff --git a/taskcluster/win-amd64-tflite-opt.yml b/taskcluster/win-amd64-tflite-opt.yml index 993f04f3..629380b7 100644 --- a/taskcluster/win-amd64-tflite-opt.yml +++ b/taskcluster/win-amd64-tflite-opt.yml @@ -8,7 +8,7 @@ build: - "node-gyp-cache" - "swig-win-amd64" - "pyenv-win-amd64" - tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r1.15.ceb46aae5836a0f648a2c3da5942af2b7d1b98bf.win/artifacts/public/home.tar.xz" + tensorflow: "https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.tensorflow.pip.r2.2.c29895fba1b9f9f48e2e54eefb024c69aa333473.win/artifacts/public/home.tar.xz" scripts: build: "taskcluster/win-build.sh --tflite" package: "taskcluster/win-package.sh" diff --git a/taskcluster/win-opt-base.tyml b/taskcluster/win-opt-base.tyml index 8d77b647..ff1aa4a5 100644 --- a/taskcluster/win-opt-base.tyml +++ b/taskcluster/win-opt-base.tyml @@ -44,7 +44,7 @@ payload: - .\msys64\usr\bin\bash.exe --login -cx "pacman -Syu --noconfirm" - echo .\msys64\usr\bin\bash.exe --login -cxe " export LC_ALL=C && - export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:$PATH\" && + export PATH=\"/c/builds/tc-workdir/msys64/usr/bin:/c/Python36:/c/Program Files/Git/bin:/c/Program Files/7-Zip/:/c/Program Files (x86)/Windows Kits/10/bin/x64/:$PATH\" && export TASKCLUSTER_ARTIFACTS=\"$USERPROFILE/public\" && export TASKCLUSTER_TASK_DIR=\"/c/builds/tc-workdir/\" && (mkdir $TASKCLUSTER_TASK_DIR || rm -fr $TASKCLUSTER_TASK_DIR/*) && cd $TASKCLUSTER_TASK_DIR && @@ -65,7 +65,7 @@ payload: $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} ; echo \"export TASKCLUSTER_TASK_EXIT_CODE=$?\" > $USERPROFILE/tc-exit.sh && cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" | cmd - /k ""C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"" x64 + /k ""C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvarsall.bat"" x64 - .\msys64\usr\bin\bash.exe --login -cxe "source $USERPROFILE/tc-exit.sh && exit $TASKCLUSTER_TASK_EXIT_CODE" @@ -82,7 +82,7 @@ payload: - file: pyenv.tar.gz content: url: ${system.pyenv.win.url} - - directory: .node-gyp + - directory: AppData/Local/node-gyp/Cache format: tar.gz content: url: ${system.node_gyp_cache.url} diff --git a/taskcluster/worker.cyml b/taskcluster/worker.cyml index 9ef5a85e..65c5c895 100644 --- a/taskcluster/worker.cyml +++ b/taskcluster/worker.cyml @@ -4,8 +4,8 @@ taskcluster: provisionerId: proj-deepspeech workerType: ci workerTypeKvm: kvm - workerTypeWin: win-b - workerTypeCuda: win-gpu + workerTypeWin: win + workerTypeCuda: win-gpu-b dockerrpi3: provisionerId: proj-deepspeech workerType: ds-rpi3