From 7d1663b1c57ee05271b738b5ac27445a2817ad0d Mon Sep 17 00:00:00 2001 From: Alexandre Lissy Date: Tue, 18 Feb 2020 09:48:47 +0100 Subject: [PATCH] Generate one-time Homebrew tarball --- .taskcluster.yml | 3 - native_client/Makefile | 2 +- taskcluster/.build.yml | 3 + taskcluster/.shared.yml | 30 ++--- taskcluster/darwin-amd64-cpu-opt.yml | 1 + taskcluster/darwin-amd64-ctc-opt.yml | 1 + taskcluster/darwin-amd64-tflite-opt.yml | 1 + taskcluster/darwin-opt-base.tyml | 54 +++------ taskcluster/homebrew-build.sh | 110 ++++++++++++++++++ taskcluster/homebrew-darwin-opt-base.tyml | 59 ++++++++++ taskcluster/homebrew-package.sh | 26 +++++ taskcluster/homebrew_builds-darwin-amd64.yml | 11 ++ taskcluster/homebrew_tests-darwin-amd64.yml | 11 ++ taskcluster/tc-all-vars.sh | 19 ++- taskcluster/tc-brew-tests.sh | 72 ------------ taskcluster/tc-build-utils.sh | 8 ++ taskcluster/tc-cpp-ds-tests-prod.sh | 4 + taskcluster/tc-cpp-ds-tests.sh | 4 + taskcluster/tc-cpp_tflite-ds-tests.sh | 4 + taskcluster/tc-cpp_tflite-tests-prod.sh | 4 + taskcluster/tc-cpp_tflite_basic-ds-tests.sh | 4 + taskcluster/test-cpp_16k-darwin-amd64-opt.yml | 4 +- .../test-cpp_16k_tflite-darwin-amd64-opt.yml | 4 +- taskcluster/test-darwin-opt-base.tyml | 25 +--- .../test-electronjs_v5.0-darwin-amd64-opt.yml | 3 +- .../test-electronjs_v6.0-darwin-amd64-opt.yml | 3 +- .../test-electronjs_v6.1-darwin-amd64-opt.yml | 3 +- .../test-electronjs_v7.0-darwin-amd64-opt.yml | 3 +- .../test-electronjs_v7.1-darwin-amd64-opt.yml | 3 +- .../test-electronjs_v8.0-darwin-amd64-opt.yml | 3 +- .../test-nodejs_10x-darwin-amd64-opt.yml | 3 +- .../test-nodejs_11x-darwin-amd64-opt.yml | 3 +- .../test-nodejs_12x-darwin-amd64-opt.yml | 3 +- .../test-nodejs_13x-darwin-amd64-opt.yml | 3 +- ...dejs_13x_multiarchpkg-darwin-amd64-opt.yml | 3 +- ...ejs_13x_multiarchpkg-darwin-tflite-opt.yml | 3 +- .../test-python_35-darwin-amd64-opt.yml | 4 +- .../test-python_36-darwin-amd64-opt.yml | 4 +- .../test-python_37-darwin-amd64-opt.yml | 4 +- .../test-python_38-darwin-amd64-opt.yml | 4 +- 40 files changed, 329 insertions(+), 187 deletions(-) create mode 100755 taskcluster/homebrew-build.sh create mode 100644 taskcluster/homebrew-darwin-opt-base.tyml create mode 100755 taskcluster/homebrew-package.sh create mode 100644 taskcluster/homebrew_builds-darwin-amd64.yml create mode 100644 taskcluster/homebrew_tests-darwin-amd64.yml delete mode 100644 taskcluster/tc-brew-tests.sh diff --git a/.taskcluster.yml b/.taskcluster.yml index 40c503a9..63e4d04a 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -26,9 +26,6 @@ tasks: "queue:route:index.project.deepspeech.*", "index:insert-task:project.deepspeech.*", "queue:scheduler-id:taskcluster-github", - "generic-worker:cache:deepspeech-homebrew-bin", - "generic-worker:cache:deepspeech-homebrew-cache", - "generic-worker:cache:deepspeech-homebrew-local", "generic-worker:cache:deepspeech-macos-pyenv", ] diff --git a/native_client/Makefile b/native_client/Makefile index 06d339f7..2b0d093e 100644 --- a/native_client/Makefile +++ b/native_client/Makefile @@ -21,7 +21,7 @@ clean: $(DEEPSPEECH_BIN): client.cc $(CXX) $(CFLAGS) $(CFLAGS_DEEPSPEECH) $(SOX_CFLAGS) client.cc $(LDFLAGS) $(SOX_LDFLAGS) ifeq ($(OS),Darwin) - install_name_tool -change $$TASKCLUSTER_TASK_DIR/homebrew/opt/sox/lib/libsox.3.dylib @rpath/libsox.3.dylib deepspeech + install_name_tool -change $$TASKCLUSTER_TASK_DIR/homebrew-builds/opt/sox/lib/libsox.3.dylib @rpath/libsox.3.dylib deepspeech install_name_tool -change bazel-out/local-opt/bin/native_client/libdeepspeech.so @rpath/libdeepspeech.so deepspeech endif diff --git a/taskcluster/.build.yml b/taskcluster/.build.yml index b457694f..c1d4bf4c 100644 --- a/taskcluster/.build.yml +++ b/taskcluster/.build.yml @@ -24,3 +24,6 @@ build: tests_cmdline: '' tensorflow_git_desc: 'TensorFlow: v1.15.0-24-gceb46aa' test_model_task: '' + homebrew: + url: '' + namespace: '' diff --git a/taskcluster/.shared.yml b/taskcluster/.shared.yml index f996f251..63ebb7b7 100644 --- a/taskcluster/.shared.yml +++ b/taskcluster/.shared.yml @@ -3,9 +3,6 @@ python: apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libpng-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev' packages_buster: apt: 'make build-essential libssl-dev libffi-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev' - brew: - setup: 'install_local_homebrew "python-ds-test" && install_pkg_local_homebrew "sox" && install_pkg_local_homebrew "readline" && install_pkg_local_homebrew "openssl" && install_pkg_local_homebrew "pkg-config"' - env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/python-ds-test.brew/bin/:$PATH"' packages_docs_bionic: apt: 'python3 python3-pip zip doxygen' electronjs: @@ -36,32 +33,25 @@ nodejs: prep_12: 'echo "deb http://deb.nodesource.com/node_12.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' prep_13: 'echo "deb http://deb.nodesource.com/node_13.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -' brew: - # Using 'nvm' from homebrew results in installing v0.33.6 which seems to - # suffer from a bug making it unable to work as of now: - # https://github.com/creationix/nvm/issues/1652 - # - # Documented workaround is to rely on 0.33.4 (bug seems to have been - # introduced by 0.33.5). - setup: 'install_local_homebrew "node-ds-test" && install_pkg_local_homebrew "sox" && export NVM_DIR=$TASKCLUSTER_TASK_DIR/.nvm/ && mkdir -p $NVM_DIR && curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash && source $NVM_DIR/nvm.sh' - env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/node-ds-test.brew/bin/:$PATH"' - prep_10: 'nvm install 10.18.1 && nvm use 10.18.1' - prep_11: 'nvm install 11.15.0 && nvm use 11.15.0' - prep_12: 'nvm install 12.14.1 && nvm use 12.14.1' - prep_13: 'nvm install 13.6.0 && nvm use 13.6.0' + prep_10: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 10.18.1' + prep_11: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 11.15.0' + prep_12: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 12.8.1' + prep_13: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 13.1.0' win: prep_10: '/usr/bin/wget.exe https://nodejs.org/dist/v10.18.1/node-v10.18.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v10.18.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v10.18.1-win-x64/:$PATH' prep_11: '/usr/bin/wget.exe https://nodejs.org/dist/v11.15.0/node-v11.15.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v11.15.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v11.15.0-win-x64/:$PATH' prep_12: '/usr/bin/wget.exe https://nodejs.org/dist/v12.14.1/node-v12.14.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v12.14.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v12.14.1-win-x64/:$PATH' prep_13: '/usr/bin/wget.exe https://nodejs.org/dist/v13.6.0/node-v13.6.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v13.6.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v13.6.0-win-x64/:$PATH' -cpp: - brew: - setup: 'install_local_homebrew "cpp-ds-test"' - packages: 'install_pkg_local_homebrew "sox"' - env: 'export EXTRA_ENV="DYLD_LIBRARY_PATH=$LOCAL_HOMEBREW_DIRECTORY/lib/:$DYLD_LIBRARY_PATH"' system: node_gyp_cache: url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.2/artifacts/public/node-gyp-cache.tar.gz' namespace: 'project.deepspeech.node-gyp-cache.2' + homebrew_builds: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.4/artifacts/public/homebrew_builds.tar.gz' + namespace: 'project.deepspeech.homebrew_builds.4' + homebrew_tests: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz' + namespace: 'project.deepspeech.homebrew_tests.4' swig: repo: "https://github.com/lissyx/swig" sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118" diff --git a/taskcluster/darwin-amd64-cpu-opt.yml b/taskcluster/darwin-amd64-cpu-opt.yml index a75a72b2..211445c9 100644 --- a/taskcluster/darwin-amd64-cpu-opt.yml +++ b/taskcluster/darwin-amd64-cpu-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-darwin-amd64" - "node-gyp-cache" + - "homebrew_builds-darwin-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx" diff --git a/taskcluster/darwin-amd64-ctc-opt.yml b/taskcluster/darwin-amd64-ctc-opt.yml index 8602efc5..a1ce32c6 100644 --- a/taskcluster/darwin-amd64-ctc-opt.yml +++ b/taskcluster/darwin-amd64-ctc-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-darwin-amd64" - "node-gyp-cache" + - "homebrew_builds-darwin-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc" diff --git a/taskcluster/darwin-amd64-tflite-opt.yml b/taskcluster/darwin-amd64-tflite-opt.yml index 898828be..c901d88a 100644 --- a/taskcluster/darwin-amd64-tflite-opt.yml +++ b/taskcluster/darwin-amd64-tflite-opt.yml @@ -3,6 +3,7 @@ build: dependencies: - "swig-darwin-amd64" - "node-gyp-cache" + - "homebrew_builds-darwin-amd64" routes: - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite" - "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite" diff --git a/taskcluster/darwin-opt-base.tyml b/taskcluster/darwin-opt-base.tyml index 7ebb1bb1..e1519bc7 100644 --- a/taskcluster/darwin-opt-base.tyml +++ b/taskcluster/darwin-opt-base.tyml @@ -29,8 +29,6 @@ routes: scopes: $if: '(event.event != "push") && (event.event != "tag")' then: - - generic-worker:cache:deepspeech-homebrew-bin - - generic-worker:cache:deepspeech-homebrew-cache - generic-worker:cache:deepspeech-macos-pyenv payload: @@ -48,26 +46,6 @@ payload: # - brew install pixz # - brew cask install java # - sudo easy_install pip - # - # gtar needed for --transform - # --transform used to rewrite symlinks - # this rewrite should take care of tranforming - # - ${system.homedir.osx}/TaskCluster/Tasks/task_[[:digit:]]\{10\}/ - # - ${system.homedir.osx}/TaskCluster/HeavyTasks/X/task_[[:digit:]]\{10\}/ - # - ${system.homedir.osx}/TaskCluster/LightTasks/X/task_[[:digit:]]\{10\}/ - # into - # - $TASKCLUSTER_TASK_DIR/ - # => goal: - # - no dangling symlink that points to broken/invalid path - # all symlinks should point to (full)path scoped only on this task - # - no file referring to anything else than $TASKCLUSTER_TASK_DIR/ - # sed to the rescue to change all text files that references the wrong - # dir and hopefully keep the tensorflow bazel cache usable - # - the huge grep -R serves this purpose (without, later build of - # native_client fails) - # - we need to remove from the grep result anything matching - # "generic-worker/" to avoid screwing the generic-worker/live_backing.log - # and this breaking the build in undebuggable ways command: - - "/bin/bash" @@ -75,31 +53,23 @@ payload: - "-cxe" - > export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && - export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && - (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && - (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) && - (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) && + export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && cd ../tc-workdir/ && (mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) && export TASKCLUSTER_TASK_DIR="$(pwd)" && export LC_ALL=C && - export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" && export MACOSX_DEPLOYMENT_TARGET=10.10 && export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && export HOMEBREW_NO_AUTO_UPDATE=1 && env && trap "export TASKCLUSTER_TASK_EXIT_CODE=$? && - (mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ || true) && - (mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ || true) && (mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ || true) && cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ && exit $TASKCLUSTER_TASK_EXIT_CODE" 0 && - (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) && + (pixz -d < $TASKCLUSTER_ORIG_TASKDIR/home.tar.xz | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) && git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client && cd $TASKCLUSTER_TASK_DIR && - $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/tc-brew.sh && - $TASKCLUSTER_TASK_DIR/homebrew/bin/brew uninstall swig && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} @@ -111,15 +81,9 @@ payload: then: { $fromNow: '6 months' } else: { $fromNow: '7 days' } - # Define node gyp cache and swig mount twice because we cannot have it - # separated from the if/then without the parser to break mounts: $if: '(event.event != "push") && (event.event != "tag")' then: - - cacheName: deepspeech-homebrew-bin - directory: homebrew/ - - cacheName: deepspeech-homebrew-cache - directory: homebrew.cache/ - cacheName: deepspeech-macos-pyenv directory: pyenv.cache/ - directory: ${system.homedir.osx}/Library/Caches/node-gyp/ @@ -130,6 +94,13 @@ payload: format: tar.gz content: url: ${system.swig.cache.darwin_amd64} + - file: home.tar.xz + content: + url: ${build.tensorflow} + - directory: ../tc-workdir/homebrew-builds + format: tar.gz + content: + url: ${system.homebrew_builds.url} else: - directory: ${system.homedir.osx}/Library/Caches/node-gyp/ format: tar.gz @@ -139,6 +110,13 @@ payload: format: tar.gz content: url: ${system.swig.cache.darwin_amd64} + - file: home.tar.xz + content: + url: ${build.tensorflow} + - directory: ../tc-workdir/homebrew-builds + format: tar.gz + content: + url: ${system.homebrew_builds.url} metadata: name: ${build.metadata.name} diff --git a/taskcluster/homebrew-build.sh b/taskcluster/homebrew-build.sh new file mode 100755 index 00000000..69dd1628 --- /dev/null +++ b/taskcluster/homebrew-build.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +set -xe + +OS=$(uname) +if [ "${OS}" != "Darwin" ]; then + echo "This should only run on OSX." + exit 1 +fi; + +flavor=$1 + +source $(dirname "$0")/tc-tests-utils.sh + +if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then + echo "No TASKCLUSTER_TASK_DIR, aborting." + exit 1 +fi + +do_prepare_homebrew() +{ + local _brew_instance=$1 + + if [ -z "${_brew_instance}" ]; then + echo "No _brew_instance, aborting." + exit 1 + fi + + export PATH=${_brew_instance}/bin:$PATH + export HOMEBREW_LOGS="${_brew_instance}/homebrew.logs/" + export HOMEBREW_CACHE="${_brew_instance}/homebrew.cache/" + export BREW_FORMULAS_COMMIT=c3f06e4f17853bea8c35742923a9c43d3a244d35 + + # Never fail on pre-existing homebrew/ directory + mkdir -p "${_brew_instance}" || true + mkdir -p "${HOMEBREW_CACHE}" || true + + # Make sure to verify there is a 'brew' binary there, otherwise install things. + if [ ! -x "${_brew_instance}/bin/brew" ]; then + curl -L ${BREW_URL} | tar xz --strip 1 -C "${_brew_instance}" + fi; + + check_homebrew + + # Force an upgrade to fetch formulae + brew search openssl + + # Then we force onto a specific well-known commit + pushd "$(brew --prefix)/Library/Taps/homebrew/homebrew-core" + git fetch origin && git checkout ${BREW_FORMULAS_COMMIT} + popd +} + +check_homebrew() +{ + echo "local brew prefix ..." + local_prefix=$(brew --prefix) + echo "${local_prefix}" + + if [ "${BUILDS_BREW}" != "${local_prefix}" ]; then + echo "Weird state:" + echo "BUILDS_BREW=${BUILDS_BREW}" + echo "local_prefix=${local_prefix}" + exit 1 + fi; +} + +install_pkg_homebrew() +{ + local pkg=$1 + (brew list --versions ${pkg} && brew upgrade --force-bottle ${pkg}) || brew install --force-bottle ${pkg} +} + +prepare_homebrew_builds() +{ + do_prepare_homebrew "${BUILDS_BREW}" + + install_pkg_homebrew "coreutils" + install_pkg_homebrew "node@12" + install_pkg_homebrew "openssl" + install_pkg_homebrew "pkg-config" + install_pkg_homebrew "pyenv-virtualenv" + install_pkg_homebrew "readline" + install_pkg_homebrew "sox" +} + +prepare_homebrew_tests() +{ + do_prepare_homebrew "${TESTS_BREW}" + + install_pkg_homebrew "nvm" + source "${TESTS_BREW}/opt/nvm/nvm.sh" + for node_ver in ${SUPPORTED_NODEJS_VERSIONS}; + do + nvm install ${node_ver} + done; + + install_pkg_homebrew "openssl" + install_pkg_homebrew "pkg-config" + install_pkg_homebrew "readline" + install_pkg_homebrew "sox" +} + +if [ "${flavor}" = "--builds" ]; then + prepare_homebrew_builds +fi; + +if [ "${flavor}" = "--tests" ]; then + prepare_homebrew_tests +fi; diff --git a/taskcluster/homebrew-darwin-opt-base.tyml b/taskcluster/homebrew-darwin-opt-base.tyml new file mode 100644 index 00000000..ac6c5af3 --- /dev/null +++ b/taskcluster/homebrew-darwin-opt-base.tyml @@ -0,0 +1,59 @@ +taskId: ${taskcluster.taskId} +provisionerId: ${taskcluster.generic.provisionerId} +workerType: ${taskcluster.generic.workerType} +taskGroupId: ${taskcluster.taskGroupId} +schedulerId: ${taskcluster.schedulerId} +dependencies: + $map: { $eval: build.dependencies } + each(b): + $eval: as_slugid(b) +created: { $fromNow: '0 sec' } +deadline: { $fromNow: '1 day' } +expires: { $fromNow: '6 months' } +scopes: + - "index:insert-task:project.deepspeech.*" + +payload: + maxRunTime: { $eval: to_int(build.maxRunTime) } + + features: + taskclusterProxy: true + + command: + - - "/bin/bash" + - "--login" + - "-cxe" + - $let: + taskIndexExpire: { $fromNow: '6 months' } + in: > + export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && + export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && + (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && + export TASKCLUSTER_TASK_DIR="$(pwd)" && + export LC_ALL=C && + export MACOSX_DEPLOYMENT_TARGET=10.10 && + export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && + env && + mkdir -p $TASKCLUSTER_ARTIFACTS/ && + swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.homebrew.url}` && + if [ "$swig_bin" != "200" ]; then + git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && + cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} && + $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} && + curl -sSL --fail -X PUT \ + -H "Content-Type: application/json" \ + -d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \ + "http://127.0.0.1:8080/index/v1/task/${build.homebrew.namespace}" + fi; + + artifacts: + - type: "directory" + path: "public/" + expires: { $fromNow: '6 months' } + +metadata: + name: ${build.metadata.name} + description: ${build.metadata.description} + owner: ${event.head.user.email} + source: ${event.head.repo.url} diff --git a/taskcluster/homebrew-package.sh b/taskcluster/homebrew-package.sh new file mode 100755 index 00000000..f3407b60 --- /dev/null +++ b/taskcluster/homebrew-package.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -xe + +OS=$(uname) +if [ "${OS}" != "Darwin" ]; then + echo "This should only run on OSX." + exit 1 +fi; + +flavor=$1 + +source $(dirname "$0")/tc-tests-utils.sh + +if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then + echo "No TASKCLUSTER_TASK_DIR, aborting." + exit 1 +fi + +if [ "${flavor}" = "--builds" ]; then + cd ${BUILDS_BREW}/ && tar -czf $TASKCLUSTER_ARTIFACTS/homebrew_builds.tar.gz . +fi; + +if [ "${flavor}" = "--tests" ]; then + cd ${TESTS_BREW}/ && tar -czf $TASKCLUSTER_ARTIFACTS/homebrew_tests.tar.gz . +fi; diff --git a/taskcluster/homebrew_builds-darwin-amd64.yml b/taskcluster/homebrew_builds-darwin-amd64.yml new file mode 100644 index 00000000..258f9ded --- /dev/null +++ b/taskcluster/homebrew_builds-darwin-amd64.yml @@ -0,0 +1,11 @@ +build: + template_file: homebrew-darwin-opt-base.tyml + homebrew: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.4/artifacts/public/homebrew_builds.tar.gz' + namespace: 'project.deepspeech.homebrew_builds.4' + scripts: + build: "taskcluster/homebrew-build.sh --builds" + package: "taskcluster/homebrew-package.sh --builds" + metadata: + name: "Builds Homebrew macOS AMD64" + description: "Setup a buildsl Homebrew for macOS/AMD64" diff --git a/taskcluster/homebrew_tests-darwin-amd64.yml b/taskcluster/homebrew_tests-darwin-amd64.yml new file mode 100644 index 00000000..68e61bdd --- /dev/null +++ b/taskcluster/homebrew_tests-darwin-amd64.yml @@ -0,0 +1,11 @@ +build: + template_file: homebrew-darwin-opt-base.tyml + homebrew: + url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz' + namespace: 'project.deepspeech.homebrew_tests.4' + scripts: + build: "taskcluster/homebrew-build.sh --tests" + package: "taskcluster/homebrew-package.sh --tests" + metadata: + name: "Tests Homebrew macOS AMD64" + description: "Setup a tests Homebrew for macOS/AMD64" diff --git a/taskcluster/tc-all-vars.sh b/taskcluster/tc-all-vars.sh index 0246b78f..02c51558 100755 --- a/taskcluster/tc-all-vars.sh +++ b/taskcluster/tc-all-vars.sh @@ -21,10 +21,21 @@ if [ "${OS}" = "Darwin" ]; then export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR} export DS_CPU_COUNT=$(sysctl hw.ncpu |cut -d' ' -f2) - # It seems chaining |export DYLD_LIBRARY_PATH=...| does not work, maybe - # because of SIP? Who knows ... - if [ ! -z "${EXTRA_ENV}" ]; then - eval "export ${EXTRA_ENV}" + export HOMEBREW_NO_AUTO_UPDATE=1 + export BREW_URL=https://github.com/Homebrew/brew/tarball/2.1.14 + + export BUILDS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-builds" + export TESTS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-tests" + + export NVM_DIR=$TESTS_BREW/.nvm/ && mkdir -p $NVM_DIR + export PKG_CONFIG_PATH="${BUILDS_BREW}/lib/pkgconfig" + + if [ -f "${BUILDS_BREW}/bin/brew" ]; then + export PATH=${BUILDS_BREW}/bin/:${BUILDS_BREW}/opt/node@12/bin:$PATH + fi; + + if [ -f "${TESTS_BREW}/bin/brew" ]; then + export PATH=${TESTS_BREW}/bin/:$PATH fi; fi; diff --git a/taskcluster/tc-brew-tests.sh b/taskcluster/tc-brew-tests.sh deleted file mode 100644 index e85d50ee..00000000 --- a/taskcluster/tc-brew-tests.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -set -xe - -OS=$(uname) -if [ "${OS}" != "Darwin" ]; then - echo "This should only run on OSX." - exit 1 -fi; - -install_local_homebrew() -{ - LOCAL_HOMEBREW_PREFIX=$1 - - if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then - echo "No TASKCLUSTER_TASK_DIR, aborting." - exit 1 - fi - - if [ -z "${LOCAL_HOMEBREW_PREFIX}" ]; then - echo "No LOCAL_HOMEBREW_PREFIX, aborting." - exit 1 - fi - - for suffix in .brew .cache .logs; - do - if [ -d "${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}${suffix}/" ]; then - echo "WARNING: Directory ${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}${suffix} already exists" - fi - done; - - export LOCAL_HOMEBREW_DIRECTORY="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.brew" - export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.logs" - export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.cache" - - mkdir -p "${LOCAL_HOMEBREW_DIRECTORY}" - mkdir -p "${HOMEBREW_CACHE}" - - curl -L https://github.com/Homebrew/brew/tarball/2.1.14 | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}" - export PATH=${LOCAL_HOMEBREW_DIRECTORY}/bin:$PATH - - if [ ! -x "${LOCAL_HOMEBREW_DIRECTORY}/bin/brew" ]; then - echo "No brew binary under ${LOCAL_HOMEBREW_DIRECTORY}, aborting" - exit 1 - fi; - - echo "local brew list (should be empty) ..." - brew list - - echo "local brew prefix ..." - local_prefix=$(brew --prefix) - echo "${local_prefix}" - - if [ "${LOCAL_HOMEBREW_DIRECTORY}" != "${local_prefix}" ]; then - echo "Weird state:" - echo "LOCAL_HOMEBREW_DIRECTORY=${LOCAL_HOMEBREW_DIRECTORY}" - echo "local_prefix=${local_prefix}" - exit 1 - fi; -} - -install_pkg_local_homebrew() -{ - pkg=$1 - - if [ ! -x "${LOCAL_HOMEBREW_DIRECTORY}/bin/brew" ]; then - echo "Cannot install $pkg: no brew binary under ${LOCAL_HOMEBREW_DIRECTORY}, aborting" - exit 1 - fi; - - (brew list --versions ${pkg} && brew upgrade --force-bottle ${pkg}) || brew install --force-bottle ${pkg} -} diff --git a/taskcluster/tc-build-utils.sh b/taskcluster/tc-build-utils.sh index 88b8f29d..0fa6b02c 100755 --- a/taskcluster/tc-build-utils.sh +++ b/taskcluster/tc-build-utils.sh @@ -52,6 +52,10 @@ do_deepspeech_python_build() setup_pyenv_virtualenv "${pyalias}" "deepspeech" virtualenv_activate "${pyalias}" "deepspeech" + python --version + which pip + which pip3 + # Set LD path because python ssl might require it LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \ @@ -117,6 +121,10 @@ do_deepspeech_decoder_build() setup_pyenv_virtualenv "${pyalias}" "deepspeech" virtualenv_activate "${pyalias}" "deepspeech" + python --version + which pip + which pip3 + # Set LD path because python ssl might require it LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \ EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \ diff --git a/taskcluster/tc-cpp-ds-tests-prod.sh b/taskcluster/tc-cpp-ds-tests-prod.sh index 9edd5673..c1dcfeb6 100644 --- a/taskcluster/tc-cpp-ds-tests-prod.sh +++ b/taskcluster/tc-cpp-ds-tests-prod.sh @@ -17,6 +17,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds" export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +if [ "${OS}" = "Darwin" ]; then + export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH +fi; + check_versions run_prod_inference_tests "${bitrate}" diff --git a/taskcluster/tc-cpp-ds-tests.sh b/taskcluster/tc-cpp-ds-tests.sh index 67d5d92f..4147941a 100644 --- a/taskcluster/tc-cpp-ds-tests.sh +++ b/taskcluster/tc-cpp-ds-tests.sh @@ -11,6 +11,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds" export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +if [ "${OS}" = "Darwin" ]; then + export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH +fi; + check_versions run_all_inference_tests diff --git a/taskcluster/tc-cpp_tflite-ds-tests.sh b/taskcluster/tc-cpp_tflite-ds-tests.sh index 313475ef..48b46ab5 100644 --- a/taskcluster/tc-cpp_tflite-ds-tests.sh +++ b/taskcluster/tc-cpp_tflite-ds-tests.sh @@ -16,6 +16,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds" export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +if [ "${OS}" = "Darwin" ]; then + export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH +fi; + check_versions run_all_inference_tests diff --git a/taskcluster/tc-cpp_tflite-tests-prod.sh b/taskcluster/tc-cpp_tflite-tests-prod.sh index 5acd4016..631eabc3 100644 --- a/taskcluster/tc-cpp_tflite-tests-prod.sh +++ b/taskcluster/tc-cpp_tflite-tests-prod.sh @@ -18,6 +18,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds" export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +if [ "${OS}" = "Darwin" ]; then + export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH +fi; + check_versions run_prodtflite_inference_tests "${bitrate}" diff --git a/taskcluster/tc-cpp_tflite_basic-ds-tests.sh b/taskcluster/tc-cpp_tflite_basic-ds-tests.sh index 7370e8a3..ddf88ec4 100644 --- a/taskcluster/tc-cpp_tflite_basic-ds-tests.sh +++ b/taskcluster/tc-cpp_tflite_basic-ds-tests.sh @@ -16,6 +16,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds" export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH +if [ "${OS}" = "Darwin" ]; then + export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH +fi; + check_versions run_tflite_basic_inference_tests diff --git a/taskcluster/test-cpp_16k-darwin-amd64-opt.yml b/taskcluster/test-cpp_16k-darwin-amd64-opt.yml index 6bc20596..45a8b3b4 100644 --- a/taskcluster/test-cpp_16k-darwin-amd64-opt.yml +++ b/taskcluster/test-cpp_16k-darwin-amd64-opt.yml @@ -3,10 +3,8 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - ${cpp.brew.setup} && ${cpp.brew.packages} && ${cpp.brew.env} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-cpp-ds-tests.sh 16k" metadata: diff --git a/taskcluster/test-cpp_16k_tflite-darwin-amd64-opt.yml b/taskcluster/test-cpp_16k_tflite-darwin-amd64-opt.yml index 44c362f8..be050c63 100644 --- a/taskcluster/test-cpp_16k_tflite-darwin-amd64-opt.yml +++ b/taskcluster/test-cpp_16k_tflite-darwin-amd64-opt.yml @@ -3,10 +3,8 @@ build: dependencies: - "darwin-amd64-tflite-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - ${cpp.brew.setup} && ${cpp.brew.packages} && ${cpp.brew.env} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-cpp_tflite_basic-ds-tests.sh 16k" metadata: diff --git a/taskcluster/test-darwin-opt-base.tyml b/taskcluster/test-darwin-opt-base.tyml index 3abcc666..9c69ac37 100644 --- a/taskcluster/test-darwin-opt-base.tyml +++ b/taskcluster/test-darwin-opt-base.tyml @@ -18,9 +18,6 @@ then: { $eval: taskcluster.github_events.pull_request } scopes: [ - "generic-worker:cache:deepspeech-homebrew-bin", - "generic-worker:cache:deepspeech-homebrew-cache", - "generic-worker:cache:deepspeech-homebrew-local", "generic-worker:cache:deepspeech-macos-pyenv", ] @@ -48,42 +45,32 @@ then: extraSystemSetup: { $eval: strip(str(build.system_setup)) } in: > export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" && - export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && - (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ && - (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) && - (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) && - (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.local/ homebrew.local/ || true) && + export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && cd ../tc-workdir/ && (mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) && export TASKCLUSTER_TASK_DIR="$(pwd)" && export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" && export LC_ALL=C && export MACOSX_DEPLOYMENT_TARGET=10.10 && export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ && - export HOMEBREW_NO_AUTO_UPDATE=1 && export PIP_DEFAULT_TIMEOUT=60 && env && trap "export TASKCLUSTER_TASK_EXIT_CODE=$? && - (mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ || true) && - (mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ || true) && - (mv $TASKCLUSTER_TASK_DIR/homebrew.local/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.local/ || true) && (mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ || true) && cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ && exit $TASKCLUSTER_TASK_EXIT_CODE" 0 && git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ && cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} && cd $TASKCLUSTER_TASK_DIR && - source $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-brew-tests.sh && ${extraSystemSetup} && + ${extraSystemSetup} && /bin/bash ${build.args.tests_cmdline} mounts: - - cacheName: deepspeech-homebrew-bin - directory: homebrew/ - - cacheName: deepspeech-homebrew-cache - directory: homebrew.cache/ - - cacheName: deepspeech-homebrew-local - directory: homebrew.local/ - cacheName: deepspeech-macos-pyenv directory: pyenv.cache/ + - directory: ../tc-workdir/homebrew-tests + format: tar.gz + content: + url: ${system.homebrew_tests.url} metadata: name: ${build.metadata.name} diff --git a/taskcluster/test-electronjs_v5.0-darwin-amd64-opt.yml b/taskcluster/test-electronjs_v5.0-darwin-amd64-opt.yml index 911eaeaf..8c2e3e16 100644 --- a/taskcluster/test-electronjs_v5.0-darwin-amd64-opt.yml +++ b/taskcluster/test-electronjs_v5.0-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env} + ${nodejs.brew.prep_12} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 5.0.6 16k" metadata: diff --git a/taskcluster/test-electronjs_v6.0-darwin-amd64-opt.yml b/taskcluster/test-electronjs_v6.0-darwin-amd64-opt.yml index 9afbb34f..08c02306 100644 --- a/taskcluster/test-electronjs_v6.0-darwin-amd64-opt.yml +++ b/taskcluster/test-electronjs_v6.0-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env} + ${nodejs.brew.prep_12} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 6.0.12 16k" metadata: diff --git a/taskcluster/test-electronjs_v6.1-darwin-amd64-opt.yml b/taskcluster/test-electronjs_v6.1-darwin-amd64-opt.yml index f512e052..3d0bf2de 100644 --- a/taskcluster/test-electronjs_v6.1-darwin-amd64-opt.yml +++ b/taskcluster/test-electronjs_v6.1-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env} + ${nodejs.brew.prep_12} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 6.1.7 16k" metadata: diff --git a/taskcluster/test-electronjs_v7.0-darwin-amd64-opt.yml b/taskcluster/test-electronjs_v7.0-darwin-amd64-opt.yml index f28c5af1..8f6340bb 100644 --- a/taskcluster/test-electronjs_v7.0-darwin-amd64-opt.yml +++ b/taskcluster/test-electronjs_v7.0-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env} + ${nodejs.brew.prep_12} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 7.0.1 16k" metadata: diff --git a/taskcluster/test-electronjs_v7.1-darwin-amd64-opt.yml b/taskcluster/test-electronjs_v7.1-darwin-amd64-opt.yml index d0a29a07..7c2fdfcd 100644 --- a/taskcluster/test-electronjs_v7.1-darwin-amd64-opt.yml +++ b/taskcluster/test-electronjs_v7.1-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env} + ${nodejs.brew.prep_12} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 7.1.2 16k" metadata: diff --git a/taskcluster/test-electronjs_v8.0-darwin-amd64-opt.yml b/taskcluster/test-electronjs_v8.0-darwin-amd64-opt.yml index aab47159..efd57f69 100644 --- a/taskcluster/test-electronjs_v8.0-darwin-amd64-opt.yml +++ b/taskcluster/test-electronjs_v8.0-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env} + ${nodejs.brew.prep_12} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 8.0.1 16k" metadata: diff --git a/taskcluster/test-nodejs_10x-darwin-amd64-opt.yml b/taskcluster/test-nodejs_10x-darwin-amd64-opt.yml index 028655ff..f1ea4abd 100644 --- a/taskcluster/test-nodejs_10x-darwin-amd64-opt.yml +++ b/taskcluster/test-nodejs_10x-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_10} && ${nodejs.brew.env} + ${nodejs.brew.prep_10} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 10.x 16k" metadata: diff --git a/taskcluster/test-nodejs_11x-darwin-amd64-opt.yml b/taskcluster/test-nodejs_11x-darwin-amd64-opt.yml index 900f9b1e..45a6535d 100644 --- a/taskcluster/test-nodejs_11x-darwin-amd64-opt.yml +++ b/taskcluster/test-nodejs_11x-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_11} && ${nodejs.brew.env} + ${nodejs.brew.prep_11} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 11.x 16k" metadata: diff --git a/taskcluster/test-nodejs_12x-darwin-amd64-opt.yml b/taskcluster/test-nodejs_12x-darwin-amd64-opt.yml index c00c22f1..7ec11441 100644 --- a/taskcluster/test-nodejs_12x-darwin-amd64-opt.yml +++ b/taskcluster/test-nodejs_12x-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env} + ${nodejs.brew.prep_12} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 12.x 16k" metadata: diff --git a/taskcluster/test-nodejs_13x-darwin-amd64-opt.yml b/taskcluster/test-nodejs_13x-darwin-amd64-opt.yml index 2c675613..3823c5ee 100644 --- a/taskcluster/test-nodejs_13x-darwin-amd64-opt.yml +++ b/taskcluster/test-nodejs_13x-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_13} && ${nodejs.brew.env} + ${nodejs.brew.prep_13} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 13.x 16k" metadata: diff --git a/taskcluster/test-nodejs_13x_multiarchpkg-darwin-amd64-opt.yml b/taskcluster/test-nodejs_13x_multiarchpkg-darwin-amd64-opt.yml index 729d329d..52c24c8f 100644 --- a/taskcluster/test-nodejs_13x_multiarchpkg-darwin-amd64-opt.yml +++ b/taskcluster/test-nodejs_13x_multiarchpkg-darwin-amd64-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "node-package-cpu" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_13} && ${nodejs.brew.env} + ${nodejs.brew.prep_13} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 13.x 16k" metadata: diff --git a/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml b/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml index fcda1593..11d64c56 100644 --- a/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml +++ b/taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml @@ -3,10 +3,11 @@ build: dependencies: - "node-package-tflite" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" system_setup: > - ${nodejs.brew.setup} && ${nodejs.brew.prep_13} && ${nodejs.brew.env} + ${nodejs.brew.prep_13} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k" metadata: diff --git a/taskcluster/test-python_35-darwin-amd64-opt.yml b/taskcluster/test-python_35-darwin-amd64-opt.yml index 5a1ece82..73ae70d8 100644 --- a/taskcluster/test-python_35-darwin-amd64-opt.yml +++ b/taskcluster/test-python_35-darwin-amd64-opt.yml @@ -3,10 +3,8 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - ${python.brew.setup} && ${python.brew.env} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.8:m 16k" metadata: diff --git a/taskcluster/test-python_36-darwin-amd64-opt.yml b/taskcluster/test-python_36-darwin-amd64-opt.yml index 11aa2c97..7675667a 100644 --- a/taskcluster/test-python_36-darwin-amd64-opt.yml +++ b/taskcluster/test-python_36-darwin-amd64-opt.yml @@ -3,10 +3,8 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - ${python.brew.setup} && ${python.brew.env} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.6.10:m 16k" metadata: diff --git a/taskcluster/test-python_37-darwin-amd64-opt.yml b/taskcluster/test-python_37-darwin-amd64-opt.yml index 4abb0c21..551f7618 100644 --- a/taskcluster/test-python_37-darwin-amd64-opt.yml +++ b/taskcluster/test-python_37-darwin-amd64-opt.yml @@ -3,10 +3,8 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - ${python.brew.setup} && ${python.brew.env} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.6:m 16k" metadata: diff --git a/taskcluster/test-python_38-darwin-amd64-opt.yml b/taskcluster/test-python_38-darwin-amd64-opt.yml index 3d990631..855d71d7 100644 --- a/taskcluster/test-python_38-darwin-amd64-opt.yml +++ b/taskcluster/test-python_38-darwin-amd64-opt.yml @@ -3,10 +3,8 @@ build: dependencies: - "darwin-amd64-cpu-opt" - "test-training_16k-linux-amd64-py36m-opt" + - "homebrew_tests-darwin-amd64" test_model_task: "test-training_16k-linux-amd64-py36m-opt" - system_setup: - > - ${python.brew.setup} && ${python.brew.env} args: tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.1: 16k" metadata: