experiments/STT
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Leverage TaskCluster-level caching for Homebrew and Python on macOS

Browse Source
This commit is contained in:
Alexandre Lissy 2019-07-11 14:55:15 +02:00
parent aa88e57403
commit cfb8f81611
8 changed files with 126 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.taskcluster.yml
View File

@ -36,7 +36,9 @@ tasks:
"queue:route:notify.irc-channel.*",
"queue:scheduler-id:taskcluster-github",
"generic-worker:cache:deepspeech-homebrew-bin",
"generic-worker:cache:deepspeech-homebrew-cache"
"generic-worker:cache:deepspeech-homebrew-cache",
"generic-worker:cache:deepspeech-homebrew-local",
"generic-worker:cache:deepspeech-macos-pyenv",
]
payload:

5
taskcluster/darwin-opt-base.tyml
View File

@ -31,6 +31,7 @@ scopes:
then:
- generic-worker:cache:deepspeech-homebrew-bin
- generic-worker:cache:deepspeech-homebrew-cache
- generic-worker:cache:deepspeech-macos-pyenv
payload:
maxRunTime: { $eval: to_int(build.maxRunTime) }
@ -82,6 +83,7 @@ payload:
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export LC_ALL=C &&
export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" &&
@ -92,6 +94,7 @@ payload:
trap "export TASKCLUSTER_TASK_EXIT_CODE=$? &&
mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ &&
mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ &&
mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ &&
cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" EXIT &&
(wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
@ -118,6 +121,8 @@ payload:
directory: homebrew/
- cacheName: deepspeech-homebrew-cache
directory: homebrew.cache/
- cacheName: deepspeech-macos-pyenv
directory: pyenv.cache/
metadata:
name: ${build.metadata.name}

13
taskcluster/tc-brew-tests.sh
View File

@ -24,15 +24,14 @@ install_local_homebrew()
for suffix in .brew .cache .logs;
do
if [ -d "${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}${suffix}/" ]; then
echo "Directory ${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}${suffix} already exists, aborting"
exit 1
if [ -d "${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}${suffix}/" ]; then
echo "WARNING: Directory ${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}${suffix} already exists"
fi
done;
export LOCAL_HOMEBREW_DIRECTORY="${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}.brew"
export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}.logs"
export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}.cache"
export LOCAL_HOMEBREW_DIRECTORY="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.brew"
export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.logs"
export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.cache"
mkdir -p "${LOCAL_HOMEBREW_DIRECTORY}"
mkdir -p "${HOMEBREW_CACHE}"
@ -69,5 +68,5 @@ install_pkg_local_homebrew()
exit 1
fi;
(brew list --versions ${pkg} && brew upgrade ${pkg}) || brew install ${pkg}
(brew list --versions ${pkg} && brew upgrade --force-bottle ${pkg}) || brew install --force-bottle ${pkg}
}

18
taskcluster/tc-python-tests-prod.sh
View File

@ -4,11 +4,17 @@ set -xe
source $(dirname "$0")/tc-tests-utils.sh
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf"
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PYENV_ROOT="${HOME}/ds-test/.pyenv"
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
@ -29,10 +35,10 @@ maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
PYENV_NAME=deepspeech-test
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}"
virtualenv_activate "${pyver}" "${PYENV_NAME}"
setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "${PYENV_NAME}"
deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type})
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
@ -41,4 +47,4 @@ run_prod_inference_tests
run_prod_concurrent_stream_tests
virtualenv_deactivate "${pyver}" "${PYENV_NAME}"
virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"

18
taskcluster/tc-python-tests.sh
View File

@ -4,11 +4,17 @@ set -xe
source $(dirname "$0")/tc-tests-utils.sh
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf"
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${PYENV_ROOT} || true
@ -23,10 +29,10 @@ maybe_ssl102_py37 ${pyver}
maybe_numpy_min_version_winamd64 ${pyver}
PYENV_NAME=deepspeech-test
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}"
virtualenv_activate "${pyver}" "${PYENV_NAME}"
setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
virtualenv_activate "${pyalias}" "${PYENV_NAME}"
deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type})
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
@ -36,4 +42,4 @@ deepspeech --version
run_all_inference_tests
virtualenv_deactivate "${pyver}" "${PYENV_NAME}"
virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"

2
taskcluster/tc-single-shot-inference.sh
View File

@ -4,7 +4,7 @@ set -xe
source $(dirname "$0")/tc-tests-utils.sh
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf"
extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
ds=$2
frozen=$2

100
taskcluster/tc-tests-utils.sh
View File

@ -611,6 +611,13 @@ install_pyenv()
popd
fi
if [ ! -d "${PYENV_ROOT}/plugins/pyenv-alias" ]; then
git clone https://github.com/s1341/pyenv-alias.git ${PYENV_ROOT}/plugins/pyenv-alias
pushd ${PYENV_ROOT}/plugins/pyenv-alias
git checkout --quiet 8896eebb5b47389249b35d21d8a5e74aa33aff08
popd
fi
eval "$(pyenv init -)"
}
@ -650,11 +657,11 @@ setup_pyenv_virtualenv()
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
echo "installing virtualenv"
PATH=${PYENV_ROOT}/versions/python.${version}/tools:${PYENV_ROOT}/versions/python.${version}/tools/Scripts:$PATH pip install virtualenv
PATH=${PYENV_ROOT}/versions/${version}/tools:${PYENV_ROOT}/versions/${version}/tools/Scripts:$PATH pip install virtualenv
echo "should setup virtualenv ${name} for ${version}"
mkdir ${PYENV_ROOT}/versions/python.${version}/envs
PATH=${PYENV_ROOT}/versions/python.${version}/tools:${PYENV_ROOT}/versions/python.${version}/tools/Scripts:$PATH virtualenv ${PYENV_ROOT}/versions/python.${version}/envs/${name}
mkdir ${PYENV_ROOT}/versions/${version}/envs
PATH=${PYENV_ROOT}/versions/${version}/tools:${PYENV_ROOT}/versions/${version}/tools/Scripts:$PATH virtualenv ${PYENV_ROOT}/versions/${version}/envs/${name}
else
pyenv virtualenv ${version} ${name}
fi
@ -671,7 +678,7 @@ virtualenv_activate()
fi;
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
source ${PYENV_ROOT}/versions/python.${version}/envs/${name}/Scripts/activate
source ${PYENV_ROOT}/versions/${version}/envs/${name}/Scripts/activate
else
source ${PYENV_ROOT}/versions/${version}/envs/${name}/bin/activate
fi
@ -690,16 +697,21 @@ virtualenv_deactivate()
deactivate
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
rm -fr ${PYENV_ROOT}/versions/python.${version}/
rm -fr ${PYENV_ROOT}/versions/${version}/
else
pyenv uninstall --force ${name}
pyenv uninstall --force ${version}
fi
}
pyenv_install()
{
local version=$1
local version_alias=$2
if [ -z "${version_alias}" ]; then
echo "WARNING, no version_alias specified, please ensure call site is okay"
version_alias=${version}
fi;
if [ -z "${PYENV_ROOT}" ]; then
echo "No PYENV_ROOT set";
@ -708,13 +720,23 @@ pyenv_install()
if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
PATH=$(cygpath ${ChocolateyInstall})/bin:$PATH nuget install python -Version ${version} -OutputDirectory ${PYENV_ROOT}/versions/
PATH=${PYENV_ROOT}/versions/python.${version}/tools/:$PATH python -m pip uninstall pip -y
PATH=${PYENV_ROOT}/versions/python.${version}/tools/:$PATH python -m ensurepip
pushd ${PYENV_ROOT}/versions/python.${version}/tools/Scripts/
mv ${PYENV_ROOT}/versions/python.${version} ${PYENV_ROOT}/versions/${version_alias}
PY_TOOLS_DIR="$(cygpath -w ${PYENV_ROOT}/versions/${version_alias}/tools/)"
TEMP=$(cygpath -w ${DS_ROOT_TASK}/tmp/) PATH=${PY_TOOLS_DIR}:$PATH python -m pip uninstall pip -y
PATH=${PY_TOOLS_DIR}:$PATH python -m ensurepip
pushd ${PYENV_ROOT}/versions/${version_alias}/tools/Scripts/
ln -s pip3.exe pip.exe
popd
else
pyenv install ${version}
# If there's already a matching directory, we should re-use it
# otherwise, pyenv install will force-rebuild
ls -hal "${PYENV_ROOT}/versions/${version_alias}/" || true
if [ ! -d "${PYENV_ROOT}/versions/${version_alias}/" ]; then
VERSION_ALIAS=${version_alias} pyenv install ${version}
fi;
fi
}
@ -1023,7 +1045,7 @@ get_python_pkg_url()
extract_python_versions()
{
# call extract_python_versions ${pyver_full} pyver pyver_pkg py_unicode_type pyconf
# call extract_python_versions ${pyver_full} pyver pyver_pkg py_unicode_type pyconf pyalias
local _pyver_full=$1
if [ -z "${_pyver_full}" ]; then
@ -1036,6 +1058,8 @@ extract_python_versions()
# 2.7.x => 27
local _pyver_pkg=$(echo "${_pyver}" | cut -d'.' -f1,2 | tr -d '.')
# UCS2 => narrow unicode
# UCS4 => wide unicode
local _py_unicode_type=$(echo "${_pyver_full}" | cut -d':' -f2)
if [ "${_py_unicode_type}" = "m" ]; then
local _pyconf="ucs2"
@ -1043,10 +1067,13 @@ extract_python_versions()
local _pyconf="ucs4"
fi;
local _pyalias="${_pyver}_${_pyconf}"
eval "${2}=${_pyver}"
eval "${3}=${_pyver_pkg}"
eval "${4}=${_py_unicode_type}"
eval "${5}=${_pyconf}"
eval "${6}=${_pyalias}"
}
do_deepspeech_python_build()
@ -1057,7 +1084,14 @@ do_deepspeech_python_build()
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/DeepSpeech/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
fi;
export PATH_WITHOUT_PYENV=${PATH}
export PATH="${PYENV_ROOT}/bin:$PATH"
install_pyenv "${PYENV_ROOT}"
@ -1074,6 +1108,8 @@ do_deepspeech_python_build()
pyver=$(echo "${pyver_conf}" | cut -d':' -f1)
pyconf=$(echo "${pyver_conf}" | cut -d':' -f2)
pyalias="${pyver}_${pyconf}"
export NUMPY_BUILD_VERSION="==1.7.0"
export NUMPY_DEP_VERSION=">=1.7.0"
@ -1081,10 +1117,12 @@ do_deepspeech_python_build()
maybe_numpy_min_version_winamd64 ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" pyenv_install ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \
pyenv_install ${pyver} ${pyalias}
setup_pyenv_virtualenv "${pyver}" "deepspeech"
virtualenv_activate "${pyver}" "deepspeech"
setup_pyenv_virtualenv "${pyalias}" "deepspeech"
virtualenv_activate "${pyalias}" "deepspeech"
# Set LD path because python ssl might require it
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
@ -1105,8 +1143,12 @@ do_deepspeech_python_build()
unset NUMPY_BUILD_VERSION
unset NUMPY_DEP_VERSION
virtualenv_deactivate "${pyver}" "deepspeech"
virtualenv_deactivate "${pyalias}" "deepspeech"
done;
# If not, and if virtualenv_deactivate does not call "pyenv uninstall ${version}"
# we get stale python2 in PATH that blocks NodeJS builds
export PATH=${PATH_WITHOUT_PYENV}
}
do_deepspeech_decoder_build()
@ -1115,7 +1157,13 @@ do_deepspeech_decoder_build()
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/DeepSpeech/.pyenv"
else
export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
fi;
export PATH="${PYENV_ROOT}/bin:$PATH"
install_pyenv "${PYENV_ROOT}"
@ -1127,15 +1175,19 @@ do_deepspeech_decoder_build()
pyver=$(echo "${pyver_conf}" | cut -d':' -f1)
pyconf=$(echo "${pyver_conf}" | cut -d':' -f2)
pyalias="${pyver}_${pyconf}"
export NUMPY_BUILD_VERSION="==1.7.0"
export NUMPY_DEP_VERSION=">=1.7.0"
maybe_ssl102_py37 ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" pyenv install ${pyver}
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \
pyenv_install ${pyver} "${pyalias}"
pyenv virtualenv ${pyver} deepspeech
source ${PYENV_ROOT}/versions/${pyver}/envs/deepspeech/bin/activate
setup_pyenv_virtualenv "${pyalias}" "deepspeech"
virtualenv_activate "${pyalias}" "deepspeech"
# Set LD path because python ssl might require it
LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
@ -1155,10 +1207,12 @@ do_deepspeech_decoder_build()
unset NUMPY_BUILD_VERSION
unset NUMPY_DEP_VERSION
deactivate
pyenv uninstall --force deepspeech
pyenv uninstall --force ${pyver}
virtualenv_deactivate "${pyalias}" "deepspeech"
done;
# If not, and if virtualenv_deactivate does not call "pyenv uninstall ${version}"
# we get stale python2 in PATH that blocks NodeJS builds
export PATH=${PATH_WITHOUT_PYENV}
}
do_deepspeech_nodejs_build()

10
taskcluster/test-darwin-opt-base.tyml
View File

@ -25,6 +25,8 @@ then:
"queue:route:notify.irc-channel.*",
"generic-worker:cache:deepspeech-homebrew-bin",
"generic-worker:cache:deepspeech-homebrew-cache",
"generic-worker:cache:deepspeech-homebrew-local",
"generic-worker:cache:deepspeech-macos-pyenv",
]
payload:
@ -57,6 +59,8 @@ then:
(mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.local/ homebrew.local/ || true) &&
(mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) &&
export TASKCLUSTER_TASK_DIR="$(pwd)" &&
export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" &&
export LC_ALL=C &&
@ -68,6 +72,8 @@ then:
trap "export TASKCLUSTER_TASK_EXIT_CODE=$? &&
mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ &&
mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ &&
mv $TASKCLUSTER_TASK_DIR/homebrew.local/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.local/ &&
mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ &&
cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" EXIT &&
git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
@ -80,6 +86,10 @@ then:
directory: homebrew/
- cacheName: deepspeech-homebrew-cache
directory: homebrew.cache/
- cacheName: deepspeech-homebrew-local
directory: homebrew.local/
- cacheName: deepspeech-macos-pyenv
directory: pyenv.cache/
metadata:
name: ${build.metadata.name}