experiments
/
STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #2766 from lissyx/homebrew-prebuilt-caches 

Generate one-time Homebrew tarball
This commit is contained in:
lissyx 2020-02-20 12:47:05 +01:00 committed by GitHub
parent fc39433f9b 7d1663b1c5
commit 234a64c6ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 329 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.taskcluster.yml
View File

@ -26,9 +26,6 @@ tasks:
"queue:route:index.project.deepspeech.*",
"index:insert-task:project.deepspeech.*",
"queue:scheduler-id:taskcluster-github",
"generic-worker:cache:deepspeech-homebrew-bin",
"generic-worker:cache:deepspeech-homebrew-cache",
"generic-worker:cache:deepspeech-homebrew-local",
"generic-worker:cache:deepspeech-macos-pyenv",
]

2
native_client/Makefile
View File

@ -21,7 +21,7 @@ clean:
$(DEEPSPEECH_BIN): client.cc
$(CXX) $(CFLAGS) $(CFLAGS_DEEPSPEECH) $(SOX_CFLAGS) client.cc $(LDFLAGS) $(SOX_LDFLAGS)
ifeq ($(OS),Darwin)
install_name_tool -change $$TASKCLUSTER_TASK_DIR/homebrew/opt/sox/lib/libsox.3.dylib @rpath/libsox.3.dylib deepspeech
install_name_tool -change $$TASKCLUSTER_TASK_DIR/homebrew-builds/opt/sox/lib/libsox.3.dylib @rpath/libsox.3.dylib deepspeech
install_name_tool -change bazel-out/local-opt/bin/native_client/libdeepspeech.so @rpath/libdeepspeech.so deepspeech
endif

3
taskcluster/.build.yml
View File

@ -24,3 +24,6 @@ build:
tests_cmdline: ''
tensorflow_git_desc: 'TensorFlow: v1.15.0-24-gceb46aa'
test_model_task: ''
homebrew:
url: ''
namespace: ''

30
taskcluster/.shared.yml
View File

@ -3,9 +3,6 @@ python:
apt: 'make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev liblzma-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libpng-dev libsox-dev libmagic-dev libgsm1-dev libltdl-dev'
packages_buster:
apt: 'make build-essential libssl-dev libffi-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev'
brew:
setup: 'install_local_homebrew "python-ds-test" && install_pkg_local_homebrew "sox" && install_pkg_local_homebrew "readline" && install_pkg_local_homebrew "openssl" && install_pkg_local_homebrew "pkg-config"'
env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/python-ds-test.brew/bin/:$PATH"'
packages_docs_bionic:
apt: 'python3 python3-pip zip doxygen'
electronjs:
@ -36,32 +33,25 @@ nodejs:
prep_12: 'echo "deb http://deb.nodesource.com/node_12.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
prep_13: 'echo "deb http://deb.nodesource.com/node_13.x buster main" > /etc/apt/sources.list.d/nodesource.list && wget -qO- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -'
brew:
# Using 'nvm' from homebrew results in installing v0.33.6 which seems to
# suffer from a bug making it unable to work as of now:
# https://github.com/creationix/nvm/issues/1652
#
# Documented workaround is to rely on 0.33.4 (bug seems to have been
# introduced by 0.33.5).
setup: 'install_local_homebrew "node-ds-test" && install_pkg_local_homebrew "sox" && export NVM_DIR=$TASKCLUSTER_TASK_DIR/.nvm/ && mkdir -p $NVM_DIR && curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash && source $NVM_DIR/nvm.sh'
env: 'export EXTRA_ENV="PATH=$TASKCLUSTER_TASK_DIR/node-ds-test.brew/bin/:$PATH"'
prep_10: 'nvm install 10.18.1 && nvm use 10.18.1'
prep_11: 'nvm install 11.15.0 && nvm use 11.15.0'
prep_12: 'nvm install 12.14.1 && nvm use 12.14.1'
prep_13: 'nvm install 13.6.0 && nvm use 13.6.0'
prep_10: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 10.18.1'
prep_11: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 11.15.0'
prep_12: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 12.8.1'
prep_13: 'export NVM_DIR=$TASKCLUSTER_TASK_DIR/homebrew-tests/.nvm/ && source "$TASKCLUSTER_TASK_DIR/homebrew-tests/opt/nvm/nvm.sh" && nvm use 13.1.0'
win:
prep_10: '/usr/bin/wget.exe https://nodejs.org/dist/v10.18.1/node-v10.18.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v10.18.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v10.18.1-win-x64/:$PATH'
prep_11: '/usr/bin/wget.exe https://nodejs.org/dist/v11.15.0/node-v11.15.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v11.15.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v11.15.0-win-x64/:$PATH'
prep_12: '/usr/bin/wget.exe https://nodejs.org/dist/v12.14.1/node-v12.14.1-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v12.14.1-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v12.14.1-win-x64/:$PATH'
prep_13: '/usr/bin/wget.exe https://nodejs.org/dist/v13.6.0/node-v13.6.0-win-x64.zip && ""C:\Program Files\7-zip\7z.exe"" x -o$TASKCLUSTER_NODE_DIR -tzip -aoa node-v13.6.0-win-x64.zip && rm node-*.zip && export PATH=$TASKCLUSTER_TASK_DIR/bin/node-v13.6.0-win-x64/:$PATH'
cpp:
brew:
setup: 'install_local_homebrew "cpp-ds-test"'
packages: 'install_pkg_local_homebrew "sox"'
env: 'export EXTRA_ENV="DYLD_LIBRARY_PATH=$LOCAL_HOMEBREW_DIRECTORY/lib/:$DYLD_LIBRARY_PATH"'
system:
node_gyp_cache:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.node-gyp-cache.2/artifacts/public/node-gyp-cache.tar.gz'
namespace: 'project.deepspeech.node-gyp-cache.2'
homebrew_builds:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.4/artifacts/public/homebrew_builds.tar.gz'
namespace: 'project.deepspeech.homebrew_builds.4'
homebrew_tests:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz'
namespace: 'project.deepspeech.homebrew_tests.4'
swig:
repo: "https://github.com/lissyx/swig"
sha1: "b5fea54d39832d1d132d7dd921b69c0c2c9d5118"

1
taskcluster/darwin-amd64-cpu-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-darwin-amd64"
- "node-gyp-cache"
- "homebrew_builds-darwin-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx"

1
taskcluster/darwin-amd64-ctc-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-darwin-amd64"
- "node-gyp-cache"
- "homebrew_builds-darwin-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-ctc"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-ctc"

1
taskcluster/darwin-amd64-tflite-opt.yml
View File

@ -3,6 +3,7 @@ build:
dependencies:
- "swig-darwin-amd64"
- "node-gyp-cache"
- "homebrew_builds-darwin-amd64"
routes:
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.osx-tflite"
- "index.project.deepspeech.deepspeech.native_client.${event.head.branchortag}.${event.head.sha}.osx-tflite"

54
taskcluster/darwin-opt-base.tyml
View File

@ -29,8 +29,6 @@ routes:
scopes:
$if: '(event.event != "push") && (event.event != "tag")'
then:
- generic-worker:cache:deepspeech-homebrew-bin
- generic-worker:cache:deepspeech-homebrew-cache
- generic-worker:cache:deepspeech-macos-pyenv
payload:
@ -48,26 +46,6 @@ payload:
# - brew install pixz
# - brew cask install java
# - sudo easy_install pip
#
# gtar needed for --transform
# --transform used to rewrite symlinks
# this rewrite should take care of tranforming
# - ${system.homedir.osx}/TaskCluster/Tasks/task_[[:digit:]]\{10\}/
# - ${system.homedir.osx}/TaskCluster/HeavyTasks/X/task_[[:digit:]]\{10\}/
# - ${system.homedir.osx}/TaskCluster/LightTasks/X/task_[[:digit:]]\{10\}/
# into
# - $TASKCLUSTER_TASK_DIR/
# => goal:
# - no dangling symlink that points to broken/invalid path
# all symlinks should point to (full)path scoped only on this task
# - no file referring to anything else than $TASKCLUSTER_TASK_DIR/
# sed to the rescue to change all text files that references the wrong
# dir and hopefully keep the tensorflow bazel cache usable
# - the huge grep -R serves this purpose (without, later build of
# native_client fails)
# - we need to remove from the grep result anything matching
# "generic-worker/" to avoid screwing the generic-worker/live_backing.log
# and this breaking the build in undebuggable ways
command:
- - "/bin/bash"
@ -75,31 +53,23 @@ payload:
- "-cxe"
- >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export LC_ALL=C &&
export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" &&
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
export HOMEBREW_NO_AUTO_UPDATE=1 &&
env &&
trap "export TASKCLUSTER_TASK_EXIT_CODE=$? &&
(mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ || true) &&
(mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ || true) &&
(mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ || true) &&
cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ &&
exit $TASKCLUSTER_TASK_EXIT_CODE" 0 &&
(wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
(pixz -d < $TASKCLUSTER_ORIG_TASKDIR/home.tar.xz | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
ln -s $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/native_client/ $TASKCLUSTER_TASK_DIR/DeepSpeech/tf/native_client &&
cd $TASKCLUSTER_TASK_DIR &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/tf/tc-brew.sh &&
$TASKCLUSTER_TASK_DIR/homebrew/bin/brew uninstall swig &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package}
@ -111,15 +81,9 @@ payload:
then: { $fromNow: '6 months' }
else: { $fromNow: '7 days' }
# Define node gyp cache and swig mount twice because we cannot have it
# separated from the if/then without the parser to break
mounts:
$if: '(event.event != "push") && (event.event != "tag")'
then:
- cacheName: deepspeech-homebrew-bin
directory: homebrew/
- cacheName: deepspeech-homebrew-cache
directory: homebrew.cache/
- cacheName: deepspeech-macos-pyenv
directory: pyenv.cache/
- directory: ${system.homedir.osx}/Library/Caches/node-gyp/
@ -130,6 +94,13 @@ payload:
format: tar.gz
content:
url: ${system.swig.cache.darwin_amd64}
- file: home.tar.xz
content:
url: ${build.tensorflow}
- directory: ../tc-workdir/homebrew-builds
format: tar.gz
content:
url: ${system.homebrew_builds.url}
else:
- directory: ${system.homedir.osx}/Library/Caches/node-gyp/
format: tar.gz
@ -139,6 +110,13 @@ payload:
format: tar.gz
content:
url: ${system.swig.cache.darwin_amd64}
- file: home.tar.xz
content:
url: ${build.tensorflow}
- directory: ../tc-workdir/homebrew-builds
format: tar.gz
content:
url: ${system.homebrew_builds.url}
metadata:
name: ${build.metadata.name}

110
taskcluster/homebrew-build.sh Executable file
View File

@ -0,0 +1,110 @@
#!/bin/bash
set -xe
OS=$(uname)
if [ "${OS}" != "Darwin" ]; then
echo "This should only run on OSX."
exit 1
fi;
flavor=$1
source $(dirname "$0")/tc-tests-utils.sh
if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then
echo "No TASKCLUSTER_TASK_DIR, aborting."
exit 1
fi
do_prepare_homebrew()
{
local _brew_instance=$1
if [ -z "${_brew_instance}" ]; then
echo "No _brew_instance, aborting."
exit 1
fi
export PATH=${_brew_instance}/bin:$PATH
export HOMEBREW_LOGS="${_brew_instance}/homebrew.logs/"
export HOMEBREW_CACHE="${_brew_instance}/homebrew.cache/"
export BREW_FORMULAS_COMMIT=c3f06e4f17853bea8c35742923a9c43d3a244d35
# Never fail on pre-existing homebrew/ directory
mkdir -p "${_brew_instance}" || true
mkdir -p "${HOMEBREW_CACHE}" || true
# Make sure to verify there is a 'brew' binary there, otherwise install things.
if [ ! -x "${_brew_instance}/bin/brew" ]; then
curl -L ${BREW_URL} | tar xz --strip 1 -C "${_brew_instance}"
fi;
check_homebrew
# Force an upgrade to fetch formulae
brew search openssl
# Then we force onto a specific well-known commit
pushd "$(brew --prefix)/Library/Taps/homebrew/homebrew-core"
git fetch origin && git checkout ${BREW_FORMULAS_COMMIT}
popd
}
check_homebrew()
{
echo "local brew prefix ..."
local_prefix=$(brew --prefix)
echo "${local_prefix}"
if [ "${BUILDS_BREW}" != "${local_prefix}" ]; then
echo "Weird state:"
echo "BUILDS_BREW=${BUILDS_BREW}"
echo "local_prefix=${local_prefix}"
exit 1
fi;
}
install_pkg_homebrew()
{
local pkg=$1
(brew list --versions ${pkg} && brew upgrade --force-bottle ${pkg}) || brew install --force-bottle ${pkg}
}
prepare_homebrew_builds()
{
do_prepare_homebrew "${BUILDS_BREW}"
install_pkg_homebrew "coreutils"
install_pkg_homebrew "node@12"
install_pkg_homebrew "openssl"
install_pkg_homebrew "pkg-config"
install_pkg_homebrew "pyenv-virtualenv"
install_pkg_homebrew "readline"
install_pkg_homebrew "sox"
}
prepare_homebrew_tests()
{
do_prepare_homebrew "${TESTS_BREW}"
install_pkg_homebrew "nvm"
source "${TESTS_BREW}/opt/nvm/nvm.sh"
for node_ver in ${SUPPORTED_NODEJS_VERSIONS};
do
nvm install ${node_ver}
done;
install_pkg_homebrew "openssl"
install_pkg_homebrew "pkg-config"
install_pkg_homebrew "readline"
install_pkg_homebrew "sox"
}
if [ "${flavor}" = "--builds" ]; then
prepare_homebrew_builds
fi;
if [ "${flavor}" = "--tests" ]; then
prepare_homebrew_tests
fi;

59
taskcluster/homebrew-darwin-opt-base.tyml Normal file
View File

@ -0,0 +1,59 @@
taskId: ${taskcluster.taskId}
provisionerId: ${taskcluster.generic.provisionerId}
workerType: ${taskcluster.generic.workerType}
taskGroupId: ${taskcluster.taskGroupId}
schedulerId: ${taskcluster.schedulerId}
dependencies:
$map: { $eval: build.dependencies }
each(b):
$eval: as_slugid(b)
created: { $fromNow: '0 sec' }
deadline: { $fromNow: '1 day' }
expires: { $fromNow: '6 months' }
scopes:
- "index:insert-task:project.deepspeech.*"
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
features:
taskclusterProxy: true
command:
- - "/bin/bash"
- "--login"
- "-cxe"
- $let:
taskIndexExpire: { $fromNow: '6 months' }
in: >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export LC_ALL=C &&
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
env &&
mkdir -p $TASKCLUSTER_ARTIFACTS/ &&
swig_bin=`curl -sSIL -o /dev/null -w "%{http_code}" ${build.homebrew.url}` &&
if [ "$swig_bin" != "200" ]; then
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.build} &&
$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/${build.scripts.package} &&
curl -sSL --fail -X PUT \
-H "Content-Type: application/json" \
-d "{\"taskId\":\"$TASK_ID\",\"rank\":0,\"expires\":\"${taskIndexExpire}\",\"data\":{}}" \
"http://127.0.0.1:8080/index/v1/task/${build.homebrew.namespace}"
fi;
artifacts:
- type: "directory"
path: "public/"
expires: { $fromNow: '6 months' }
metadata:
name: ${build.metadata.name}
description: ${build.metadata.description}
owner: ${event.head.user.email}
source: ${event.head.repo.url}

26
taskcluster/homebrew-package.sh Executable file
View File

@ -0,0 +1,26 @@
#!/bin/bash
set -xe
OS=$(uname)
if [ "${OS}" != "Darwin" ]; then
echo "This should only run on OSX."
exit 1
fi;
flavor=$1
source $(dirname "$0")/tc-tests-utils.sh
if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then
echo "No TASKCLUSTER_TASK_DIR, aborting."
exit 1
fi
if [ "${flavor}" = "--builds" ]; then
cd ${BUILDS_BREW}/ && tar -czf $TASKCLUSTER_ARTIFACTS/homebrew_builds.tar.gz .
fi;
if [ "${flavor}" = "--tests" ]; then
cd ${TESTS_BREW}/ && tar -czf $TASKCLUSTER_ARTIFACTS/homebrew_tests.tar.gz .
fi;

11
taskcluster/homebrew_builds-darwin-amd64.yml Normal file
View File

@ -0,0 +1,11 @@
build:
template_file: homebrew-darwin-opt-base.tyml
homebrew:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_builds.4/artifacts/public/homebrew_builds.tar.gz'
namespace: 'project.deepspeech.homebrew_builds.4'
scripts:
build: "taskcluster/homebrew-build.sh --builds"
package: "taskcluster/homebrew-package.sh --builds"
metadata:
name: "Builds Homebrew macOS AMD64"
description: "Setup a buildsl Homebrew for macOS/AMD64"

11
taskcluster/homebrew_tests-darwin-amd64.yml Normal file
View File

@ -0,0 +1,11 @@
build:
template_file: homebrew-darwin-opt-base.tyml
homebrew:
url: 'https://community-tc.services.mozilla.com/api/index/v1/task/project.deepspeech.homebrew_tests.4/artifacts/public/homebrew_tests.tar.gz'
namespace: 'project.deepspeech.homebrew_tests.4'
scripts:
build: "taskcluster/homebrew-build.sh --tests"
package: "taskcluster/homebrew-package.sh --tests"
metadata:
name: "Tests Homebrew macOS AMD64"
description: "Setup a tests Homebrew for macOS/AMD64"

19
taskcluster/tc-all-vars.sh
View File

@ -21,10 +21,21 @@ if [ "${OS}" = "Darwin" ]; then
export DS_ROOT_TASK=${TASKCLUSTER_TASK_DIR}
export DS_CPU_COUNT=$(sysctl hw.ncpu |cut -d' ' -f2)
# It seems chaining |export DYLD_LIBRARY_PATH=...| does not work, maybe
# because of SIP? Who knows ...
if [ ! -z "${EXTRA_ENV}" ]; then
eval "export ${EXTRA_ENV}"
export HOMEBREW_NO_AUTO_UPDATE=1
export BREW_URL=https://github.com/Homebrew/brew/tarball/2.1.14
export BUILDS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-builds"
export TESTS_BREW="${TASKCLUSTER_TASK_DIR}/homebrew-tests"
export NVM_DIR=$TESTS_BREW/.nvm/ && mkdir -p $NVM_DIR
export PKG_CONFIG_PATH="${BUILDS_BREW}/lib/pkgconfig"
if [ -f "${BUILDS_BREW}/bin/brew" ]; then
export PATH=${BUILDS_BREW}/bin/:${BUILDS_BREW}/opt/node@12/bin:$PATH
fi;
if [ -f "${TESTS_BREW}/bin/brew" ]; then
export PATH=${TESTS_BREW}/bin/:$PATH
fi;
fi;

72
taskcluster/tc-brew-tests.sh
View File

@ -1,72 +0,0 @@
#!/bin/bash
set -xe
OS=$(uname)
if [ "${OS}" != "Darwin" ]; then
echo "This should only run on OSX."
exit 1
fi;
install_local_homebrew()
{
LOCAL_HOMEBREW_PREFIX=$1
if [ -z "${TASKCLUSTER_TASK_DIR}" ]; then
echo "No TASKCLUSTER_TASK_DIR, aborting."
exit 1
fi
if [ -z "${LOCAL_HOMEBREW_PREFIX}" ]; then
echo "No LOCAL_HOMEBREW_PREFIX, aborting."
exit 1
fi
for suffix in .brew .cache .logs;
do
if [ -d "${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}${suffix}/" ]; then
echo "WARNING: Directory ${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}${suffix} already exists"
fi
done;
export LOCAL_HOMEBREW_DIRECTORY="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.brew"
export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.logs"
export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.cache"
mkdir -p "${LOCAL_HOMEBREW_DIRECTORY}"
mkdir -p "${HOMEBREW_CACHE}"
curl -L https://github.com/Homebrew/brew/tarball/2.1.14 | tar xz --strip 1 -C "${LOCAL_HOMEBREW_DIRECTORY}"
export PATH=${LOCAL_HOMEBREW_DIRECTORY}/bin:$PATH
if [ ! -x "${LOCAL_HOMEBREW_DIRECTORY}/bin/brew" ]; then
echo "No brew binary under ${LOCAL_HOMEBREW_DIRECTORY}, aborting"
exit 1
fi;
echo "local brew list (should be empty) ..."
brew list
echo "local brew prefix ..."
local_prefix=$(brew --prefix)
echo "${local_prefix}"
if [ "${LOCAL_HOMEBREW_DIRECTORY}" != "${local_prefix}" ]; then
echo "Weird state:"
echo "LOCAL_HOMEBREW_DIRECTORY=${LOCAL_HOMEBREW_DIRECTORY}"
echo "local_prefix=${local_prefix}"
exit 1
fi;
}
install_pkg_local_homebrew()
{
pkg=$1
if [ ! -x "${LOCAL_HOMEBREW_DIRECTORY}/bin/brew" ]; then
echo "Cannot install $pkg: no brew binary under ${LOCAL_HOMEBREW_DIRECTORY}, aborting"
exit 1
fi;
(brew list --versions ${pkg} && brew upgrade --force-bottle ${pkg}) || brew install --force-bottle ${pkg}
}

8
taskcluster/tc-build-utils.sh
View File

@ -52,6 +52,10 @@ do_deepspeech_python_build()
setup_pyenv_virtualenv "${pyalias}" "deepspeech"
virtualenv_activate "${pyalias}" "deepspeech"
python --version
which pip
which pip3
# Set LD path because python ssl might require it
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \
@ -117,6 +121,10 @@ do_deepspeech_decoder_build()
setup_pyenv_virtualenv "${pyalias}" "deepspeech"
virtualenv_activate "${pyalias}" "deepspeech"
python --version
which pip
which pip3
# Set LD path because python ssl might require it
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
EXTRA_CFLAGS="${EXTRA_LOCAL_CFLAGS}" \

4
taskcluster/tc-cpp-ds-tests-prod.sh
View File

@ -17,6 +17,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds"
export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
if [ "${OS}" = "Darwin" ]; then
export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH
fi;
check_versions
run_prod_inference_tests "${bitrate}"

4
taskcluster/tc-cpp-ds-tests.sh
View File

@ -11,6 +11,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds"
export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
if [ "${OS}" = "Darwin" ]; then
export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH
fi;
check_versions
run_all_inference_tests

4
taskcluster/tc-cpp_tflite-ds-tests.sh
View File

@ -16,6 +16,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds"
export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
if [ "${OS}" = "Darwin" ]; then
export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH
fi;
check_versions
run_all_inference_tests

4
taskcluster/tc-cpp_tflite-tests-prod.sh
View File

@ -18,6 +18,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds"
export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
if [ "${OS}" = "Darwin" ]; then
export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH
fi;
check_versions
run_prodtflite_inference_tests "${bitrate}"

4
taskcluster/tc-cpp_tflite_basic-ds-tests.sh
View File

@ -16,6 +16,10 @@ download_material "${TASKCLUSTER_TMP_DIR}/ds"
export PATH=${TASKCLUSTER_TMP_DIR}/ds/:$PATH
if [ "${OS}" = "Darwin" ]; then
export DYLD_LIBRARY_PATH=$TESTS_BREW/lib/:$DYLD_LIBRARY_PATH
fi;
check_versions
run_tflite_basic_inference_tests

4
taskcluster/test-cpp_16k-darwin-amd64-opt.yml
View File

@ -3,10 +3,8 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${cpp.brew.setup} && ${cpp.brew.packages} && ${cpp.brew.env}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-cpp-ds-tests.sh 16k"
metadata:

4
taskcluster/test-cpp_16k_tflite-darwin-amd64-opt.yml
View File

@ -3,10 +3,8 @@ build:
dependencies:
- "darwin-amd64-tflite-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${cpp.brew.setup} && ${cpp.brew.packages} && ${cpp.brew.env}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-cpp_tflite_basic-ds-tests.sh 16k"
metadata:

25
taskcluster/test-darwin-opt-base.tyml
View File

@ -18,9 +18,6 @@ then:
{ $eval: taskcluster.github_events.pull_request }
scopes: [
"generic-worker:cache:deepspeech-homebrew-bin",
"generic-worker:cache:deepspeech-homebrew-cache",
"generic-worker:cache:deepspeech-homebrew-local",
"generic-worker:cache:deepspeech-macos-pyenv",
]
@ -48,42 +45,32 @@ then:
extraSystemSetup: { $eval: strip(str(build.system_setup)) }
in: >
export TASKCLUSTER_ARTIFACTS="$(pwd)/public/" &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" &&
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.local/ homebrew.local/ || true) &&
export TASKCLUSTER_ORIG_TASKDIR="$(pwd)" && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" &&
export LC_ALL=C &&
export MACOSX_DEPLOYMENT_TARGET=10.10 &&
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX10.14.sdk/ &&
export HOMEBREW_NO_AUTO_UPDATE=1 &&
export PIP_DEFAULT_TIMEOUT=60 &&
env &&
trap "export TASKCLUSTER_TASK_EXIT_CODE=$? &&
(mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ || true) &&
(mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ || true) &&
(mv $TASKCLUSTER_TASK_DIR/homebrew.local/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.local/ || true) &&
(mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ || true) &&
cd $TASKCLUSTER_ORIG_TASKDIR/ && rm -fr $TASKCLUSTER_TASK_DIR/ &&
exit $TASKCLUSTER_TASK_EXIT_CODE" 0 &&
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
cd $TASKCLUSTER_TASK_DIR &&
source $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-brew-tests.sh && ${extraSystemSetup} &&
${extraSystemSetup} &&
/bin/bash ${build.args.tests_cmdline}
mounts:
- cacheName: deepspeech-homebrew-bin
directory: homebrew/
- cacheName: deepspeech-homebrew-cache
directory: homebrew.cache/
- cacheName: deepspeech-homebrew-local
directory: homebrew.local/
- cacheName: deepspeech-macos-pyenv
directory: pyenv.cache/
- directory: ../tc-workdir/homebrew-tests
format: tar.gz
content:
url: ${system.homebrew_tests.url}
metadata:
name: ${build.metadata.name}

3
taskcluster/test-electronjs_v5.0-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env}
${nodejs.brew.prep_12}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 5.0.6 16k"
metadata:

3
taskcluster/test-electronjs_v6.0-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env}
${nodejs.brew.prep_12}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 6.0.12 16k"
metadata:

3
taskcluster/test-electronjs_v6.1-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env}
${nodejs.brew.prep_12}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 6.1.7 16k"
metadata:

3
taskcluster/test-electronjs_v7.0-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env}
${nodejs.brew.prep_12}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 7.0.1 16k"
metadata:

3
taskcluster/test-electronjs_v7.1-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env}
${nodejs.brew.prep_12}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 7.1.2 16k"
metadata:

3
taskcluster/test-electronjs_v8.0-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env}
${nodejs.brew.prep_12}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-electron-tests.sh 12.x 8.0.1 16k"
metadata:

3
taskcluster/test-nodejs_10x-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_10} && ${nodejs.brew.env}
${nodejs.brew.prep_10}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 10.x 16k"
metadata:

3
taskcluster/test-nodejs_11x-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_11} && ${nodejs.brew.env}
${nodejs.brew.prep_11}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 11.x 16k"
metadata:

3
taskcluster/test-nodejs_12x-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_12} && ${nodejs.brew.env}
${nodejs.brew.prep_12}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 12.x 16k"
metadata:

3
taskcluster/test-nodejs_13x-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_13} && ${nodejs.brew.env}
${nodejs.brew.prep_13}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 13.x 16k"
metadata:

3
taskcluster/test-nodejs_13x_multiarchpkg-darwin-amd64-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "node-package-cpu"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_13} && ${nodejs.brew.env}
${nodejs.brew.prep_13}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node-tests.sh 13.x 16k"
metadata:

3
taskcluster/test-nodejs_13x_multiarchpkg-darwin-tflite-opt.yml
View File

@ -3,10 +3,11 @@ build:
dependencies:
- "node-package-tflite"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${nodejs.brew.setup} && ${nodejs.brew.prep_13} && ${nodejs.brew.env}
${nodejs.brew.prep_13}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-node_tflite-tests.sh 13.x 16k"
metadata:

4
taskcluster/test-python_35-darwin-amd64-opt.yml
View File

@ -3,10 +3,8 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${python.brew.setup} && ${python.brew.env}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.5.8:m 16k"
metadata:

4
taskcluster/test-python_36-darwin-amd64-opt.yml
View File

@ -3,10 +3,8 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${python.brew.setup} && ${python.brew.env}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.6.10:m 16k"
metadata:

4
taskcluster/test-python_37-darwin-amd64-opt.yml
View File

@ -3,10 +3,8 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${python.brew.setup} && ${python.brew.env}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.7.6:m 16k"
metadata:

4
taskcluster/test-python_38-darwin-amd64-opt.yml
View File

@ -3,10 +3,8 @@ build:
dependencies:
- "darwin-amd64-cpu-opt"
- "test-training_16k-linux-amd64-py36m-opt"
- "homebrew_tests-darwin-amd64"
test_model_task: "test-training_16k-linux-amd64-py36m-opt"
system_setup:
>
${python.brew.setup} && ${python.brew.env}
args:
tests_cmdline: "$TASKCLUSTER_TASK_DIR/DeepSpeech/ds/taskcluster/tc-python-tests.sh 3.8.1: 16k"
metadata: