Merge pull request #2239 from lissyx/try-bottle

Use TaskCluster caching for Python and Homebrew setup on macOS

.taskcluster.yml

@@ -36,7 +36,9 @@ tasks:
       "queue:route:notify.irc-channel.*",
       "queue:scheduler-id:taskcluster-github",
       "generic-worker:cache:deepspeech-homebrew-bin",
-      "generic-worker:cache:deepspeech-homebrew-cache"
+      "generic-worker:cache:deepspeech-homebrew-cache",
+      "generic-worker:cache:deepspeech-homebrew-local",
+      "generic-worker:cache:deepspeech-macos-pyenv",
     ]
 
     payload:

taskcluster/darwin-opt-base.tyml

@@ -31,6 +31,7 @@ scopes:
   then:
     - generic-worker:cache:deepspeech-homebrew-bin
     - generic-worker:cache:deepspeech-homebrew-cache
+    - generic-worker:cache:deepspeech-macos-pyenv
 
 payload:
   maxRunTime: { $eval: to_int(build.maxRunTime) }
@@ -82,6 +83,7 @@ payload:
         (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
         (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) &&
         (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) &&
+        (mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) &&
         export TASKCLUSTER_TASK_DIR="$(pwd)" &&
         export LC_ALL=C &&
         export PKG_CONFIG_PATH="$TASKCLUSTER_TASK_DIR/homebrew/lib/pkgconfig" &&
@@ -92,6 +94,7 @@ payload:
         trap "export TASKCLUSTER_TASK_EXIT_CODE=$? &&
         mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ &&
         mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ &&
+        mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ &&
         cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" EXIT &&
         (wget -O - $TENSORFLOW_BUILD_ARTIFACT | pixz -d | gtar -C $TASKCLUSTER_TASK_DIR -xf - ) &&
         git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
@@ -118,6 +121,8 @@ payload:
         directory: homebrew/
       - cacheName: deepspeech-homebrew-cache
         directory: homebrew.cache/
+      - cacheName: deepspeech-macos-pyenv
+        directory: pyenv.cache/
 
 metadata:
   name: ${build.metadata.name}

taskcluster/tc-brew-tests.sh

@@ -24,15 +24,14 @@ install_local_homebrew()
 
     for suffix in .brew .cache .logs;
     do
-        if [ -d "${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}${suffix}/" ]; then
+        if [ -d "${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}${suffix}/" ]; then
-            echo "Directory ${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}${suffix} already exists, aborting"
+            echo "WARNING: Directory ${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}${suffix} already exists"
-            exit 1
         fi
     done;
 
-    export LOCAL_HOMEBREW_DIRECTORY="${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}.brew"
+    export LOCAL_HOMEBREW_DIRECTORY="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.brew"
-    export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}.logs"
+    export HOMEBREW_LOGS="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.logs"
-    export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/${LOCAL_HOMEBREW_PREFIX}.cache"
+    export HOMEBREW_CACHE="${TASKCLUSTER_TASK_DIR}/homebrew.local/${LOCAL_HOMEBREW_PREFIX}.cache"
 
     mkdir -p "${LOCAL_HOMEBREW_DIRECTORY}"
     mkdir -p "${HOMEBREW_CACHE}"
@@ -69,5 +68,5 @@ install_pkg_local_homebrew()
         exit 1
     fi;
 
-    (brew list --versions ${pkg} && brew upgrade ${pkg}) || brew install ${pkg}
+    (brew list --versions ${pkg} && brew upgrade --force-bottle ${pkg}) || brew install --force-bottle ${pkg}
 }

taskcluster/tc-python-tests-prod.sh

@@ -4,11 +4,17 @@ set -xe
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf"
+extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
 
 unset PYTHON_BIN_PATH
 unset PYTHONPATH
-export PYENV_ROOT="${HOME}/ds-test/.pyenv"
+
+if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
+  export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
+else
+  export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
+fi;
+
 export PATH="${PYENV_ROOT}/bin:$PATH"
 
 mkdir -p ${PYENV_ROOT} || true
@@ -29,10 +35,10 @@ maybe_ssl102_py37 ${pyver}
 maybe_numpy_min_version_winamd64 ${pyver}
 
 PYENV_NAME=deepspeech-test
-LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver}
+LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
 
-setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}"
+setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
-virtualenv_activate "${pyver}" "${PYENV_NAME}"
+virtualenv_activate "${pyalias}" "${PYENV_NAME}"
 
 deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type})
 LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
@@ -41,4 +47,4 @@ run_prod_inference_tests
 
 run_prod_concurrent_stream_tests
 
-virtualenv_deactivate "${pyver}" "${PYENV_NAME}"
+virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"

taskcluster/tc-python-tests.sh

@@ -4,11 +4,17 @@ set -xe
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf"
+extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
 
 unset PYTHON_BIN_PATH
 unset PYTHONPATH
-export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
+
+if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
+  export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/ds-test/.pyenv"
+else
+  export PYENV_ROOT="${DS_ROOT_TASK}/ds-test/.pyenv"
+fi;
+
 export PATH="${PYENV_ROOT}/bin:$PATH"
 
 mkdir -p ${PYENV_ROOT} || true
@@ -23,10 +29,10 @@ maybe_ssl102_py37 ${pyver}
 maybe_numpy_min_version_winamd64 ${pyver}
 
 PYENV_NAME=deepspeech-test
-LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver}
+LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL} ${EXTRA_PYTHON_CONFIGURE_OPTS}" pyenv_install ${pyver} ${pyalias}
 
-setup_pyenv_virtualenv "${pyver}" "${PYENV_NAME}"
+setup_pyenv_virtualenv "${pyalias}" "${PYENV_NAME}"
-virtualenv_activate "${pyver}" "${PYENV_NAME}"
+virtualenv_activate "${pyalias}" "${PYENV_NAME}"
 
 deepspeech_pkg_url=$(get_python_pkg_url ${pyver_pkg} ${py_unicode_type})
 LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH pip install --verbose --only-binary :all: ${PY37_SOURCE_PACKAGE} --upgrade ${deepspeech_pkg_url} | cat
@@ -36,4 +42,4 @@ deepspeech --version
 
 run_all_inference_tests
 
-virtualenv_deactivate "${pyver}" "${PYENV_NAME}"
+virtualenv_deactivate "${pyalias}" "${PYENV_NAME}"

taskcluster/tc-single-shot-inference.sh

@@ -4,7 +4,7 @@ set -xe
 
 source $(dirname "$0")/tc-tests-utils.sh
 
-extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf"
+extract_python_versions "$1" "pyver" "pyver_pkg" "py_unicode_type" "pyconf" "pyalias"
 
 ds=$2
 frozen=$2

taskcluster/tc-tests-utils.sh

@@ -611,6 +611,13 @@ install_pyenv()
     popd
   fi
 
+  if [ ! -d "${PYENV_ROOT}/plugins/pyenv-alias" ]; then
+    git clone https://github.com/s1341/pyenv-alias.git ${PYENV_ROOT}/plugins/pyenv-alias
+    pushd ${PYENV_ROOT}/plugins/pyenv-alias
+      git checkout --quiet 8896eebb5b47389249b35d21d8a5e74aa33aff08
+    popd
+  fi
+
   eval "$(pyenv init -)"
 }
 
@@ -650,11 +657,11 @@ setup_pyenv_virtualenv()
 
   if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
     echo "installing virtualenv"
-    PATH=${PYENV_ROOT}/versions/python.${version}/tools:${PYENV_ROOT}/versions/python.${version}/tools/Scripts:$PATH pip install virtualenv
+    PATH=${PYENV_ROOT}/versions/${version}/tools:${PYENV_ROOT}/versions/${version}/tools/Scripts:$PATH pip install virtualenv
 
     echo "should setup virtualenv ${name} for ${version}"
-    mkdir ${PYENV_ROOT}/versions/python.${version}/envs
+    mkdir ${PYENV_ROOT}/versions/${version}/envs
-    PATH=${PYENV_ROOT}/versions/python.${version}/tools:${PYENV_ROOT}/versions/python.${version}/tools/Scripts:$PATH virtualenv ${PYENV_ROOT}/versions/python.${version}/envs/${name}
+    PATH=${PYENV_ROOT}/versions/${version}/tools:${PYENV_ROOT}/versions/${version}/tools/Scripts:$PATH virtualenv ${PYENV_ROOT}/versions/${version}/envs/${name}
   else
     pyenv virtualenv ${version} ${name}
   fi
@@ -671,7 +678,7 @@ virtualenv_activate()
   fi;
 
   if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
-    source ${PYENV_ROOT}/versions/python.${version}/envs/${name}/Scripts/activate
+    source ${PYENV_ROOT}/versions/${version}/envs/${name}/Scripts/activate
   else
     source ${PYENV_ROOT}/versions/${version}/envs/${name}/bin/activate
   fi
@@ -690,16 +697,21 @@ virtualenv_deactivate()
   deactivate
 
   if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
-    rm -fr ${PYENV_ROOT}/versions/python.${version}/
+    rm -fr ${PYENV_ROOT}/versions/${version}/
   else
     pyenv uninstall --force ${name}
-    pyenv uninstall --force ${version}
   fi
 }
 
 pyenv_install()
 {
   local version=$1
+  local version_alias=$2
+
+  if [ -z "${version_alias}" ]; then
+    echo "WARNING, no version_alias specified, please ensure call site is okay"
+    version_alias=${version}
+  fi;
 
   if [ -z "${PYENV_ROOT}" ]; then
     echo "No PYENV_ROOT set";
@@ -708,13 +720,23 @@ pyenv_install()
 
   if [ "${OS}" = "${TC_MSYS_VERSION}" ]; then
     PATH=$(cygpath ${ChocolateyInstall})/bin:$PATH nuget install python -Version ${version} -OutputDirectory ${PYENV_ROOT}/versions/
-    PATH=${PYENV_ROOT}/versions/python.${version}/tools/:$PATH python -m pip uninstall pip -y
+
-    PATH=${PYENV_ROOT}/versions/python.${version}/tools/:$PATH python -m ensurepip
+    mv ${PYENV_ROOT}/versions/python.${version} ${PYENV_ROOT}/versions/${version_alias}
-    pushd ${PYENV_ROOT}/versions/python.${version}/tools/Scripts/
+
+    PY_TOOLS_DIR="$(cygpath -w ${PYENV_ROOT}/versions/${version_alias}/tools/)"
+    TEMP=$(cygpath -w ${DS_ROOT_TASK}/tmp/) PATH=${PY_TOOLS_DIR}:$PATH python -m pip uninstall pip -y
+    PATH=${PY_TOOLS_DIR}:$PATH python -m ensurepip
+
+    pushd ${PYENV_ROOT}/versions/${version_alias}/tools/Scripts/
       ln -s pip3.exe pip.exe
     popd
   else
-    pyenv install ${version}
+    # If there's already a matching directory, we should re-use it
+    # otherwise, pyenv install will force-rebuild
+    ls -hal "${PYENV_ROOT}/versions/${version_alias}/" || true
+    if [ ! -d "${PYENV_ROOT}/versions/${version_alias}/" ]; then
+      VERSION_ALIAS=${version_alias} pyenv install ${version}
+    fi;
   fi
 }
 
@@ -1023,7 +1045,7 @@ get_python_pkg_url()
 
 extract_python_versions()
 {
-  # call extract_python_versions ${pyver_full} pyver pyver_pkg py_unicode_type pyconf
+  # call extract_python_versions ${pyver_full} pyver pyver_pkg py_unicode_type pyconf pyalias
   local _pyver_full=$1
 
   if [ -z "${_pyver_full}" ]; then
@@ -1036,6 +1058,8 @@ extract_python_versions()
   # 2.7.x => 27
   local _pyver_pkg=$(echo "${_pyver}" | cut -d'.' -f1,2 | tr -d '.')
 
+  # UCS2 => narrow unicode
+  # UCS4 => wide unicode
   local _py_unicode_type=$(echo "${_pyver_full}" | cut -d':' -f2)
   if [ "${_py_unicode_type}" = "m" ]; then
     local _pyconf="ucs2"
@@ -1043,10 +1067,13 @@ extract_python_versions()
     local _pyconf="ucs4"
   fi;
 
+  local _pyalias="${_pyver}_${_pyconf}"
+
   eval "${2}=${_pyver}"
   eval "${3}=${_pyver_pkg}"
   eval "${4}=${_py_unicode_type}"
   eval "${5}=${_pyconf}"
+  eval "${6}=${_pyalias}"
 }
 
 do_deepspeech_python_build()
@@ -1057,7 +1084,14 @@ do_deepspeech_python_build()
 
   unset PYTHON_BIN_PATH
   unset PYTHONPATH
-  export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
+
+  if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
+    export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/DeepSpeech/.pyenv"
+  else
+    export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
+  fi;
+
+  export PATH_WITHOUT_PYENV=${PATH}
   export PATH="${PYENV_ROOT}/bin:$PATH"
 
   install_pyenv "${PYENV_ROOT}"
@@ -1074,6 +1108,8 @@ do_deepspeech_python_build()
     pyver=$(echo "${pyver_conf}" | cut -d':' -f1)
     pyconf=$(echo "${pyver_conf}" | cut -d':' -f2)
 
+    pyalias="${pyver}_${pyconf}"
+
     export NUMPY_BUILD_VERSION="==1.7.0"
     export NUMPY_DEP_VERSION=">=1.7.0"
 
@@ -1081,10 +1117,12 @@ do_deepspeech_python_build()
 
     maybe_numpy_min_version_winamd64 ${pyver}
 
-    LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" pyenv_install ${pyver}
+    LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
+        PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \
+        pyenv_install ${pyver} ${pyalias}
 
-    setup_pyenv_virtualenv "${pyver}" "deepspeech"
+    setup_pyenv_virtualenv "${pyalias}" "deepspeech"
-    virtualenv_activate "${pyver}" "deepspeech"
+    virtualenv_activate "${pyalias}" "deepspeech"
 
     # Set LD path because python ssl might require it
     LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
@@ -1105,8 +1143,12 @@ do_deepspeech_python_build()
     unset NUMPY_BUILD_VERSION
     unset NUMPY_DEP_VERSION
 
-    virtualenv_deactivate "${pyver}" "deepspeech"
+    virtualenv_deactivate "${pyalias}" "deepspeech"
   done;
+
+  # If not, and if virtualenv_deactivate does not call "pyenv uninstall ${version}"
+  # we get stale python2 in PATH that blocks NodeJS builds
+  export PATH=${PATH_WITHOUT_PYENV}
 }
 
 do_deepspeech_decoder_build()
@@ -1115,7 +1157,13 @@ do_deepspeech_decoder_build()
 
   unset PYTHON_BIN_PATH
   unset PYTHONPATH
-  export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
+
+  if [ -d "${DS_ROOT_TASK}/pyenv.cache/" ]; then
+    export PYENV_ROOT="${DS_ROOT_TASK}/pyenv.cache/DeepSpeech/.pyenv"
+  else
+    export PYENV_ROOT="${DS_ROOT_TASK}/DeepSpeech/.pyenv"
+  fi;
+
   export PATH="${PYENV_ROOT}/bin:$PATH"
 
   install_pyenv "${PYENV_ROOT}"
@@ -1127,15 +1175,19 @@ do_deepspeech_decoder_build()
     pyver=$(echo "${pyver_conf}" | cut -d':' -f1)
     pyconf=$(echo "${pyver_conf}" | cut -d':' -f2)
 
+    pyalias="${pyver}_${pyconf}"
+
     export NUMPY_BUILD_VERSION="==1.7.0"
     export NUMPY_DEP_VERSION=">=1.7.0"
 
     maybe_ssl102_py37 ${pyver}
 
-    LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" pyenv install ${pyver}
+    LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
+        PYTHON_CONFIGURE_OPTS="--enable-unicode=${pyconf} ${PY37_OPENSSL}" \
+        pyenv_install ${pyver} "${pyalias}"
 
-    pyenv virtualenv ${pyver} deepspeech
+    setup_pyenv_virtualenv "${pyalias}" "deepspeech"
-    source ${PYENV_ROOT}/versions/${pyver}/envs/deepspeech/bin/activate
+    virtualenv_activate "${pyalias}" "deepspeech"
 
     # Set LD path because python ssl might require it
     LD_LIBRARY_PATH=${PY37_LDPATH}:$LD_LIBRARY_PATH \
@@ -1155,10 +1207,12 @@ do_deepspeech_decoder_build()
     unset NUMPY_BUILD_VERSION
     unset NUMPY_DEP_VERSION
 
-    deactivate
+    virtualenv_deactivate "${pyalias}" "deepspeech"
-    pyenv uninstall --force deepspeech
-    pyenv uninstall --force ${pyver}
   done;
+
+  # If not, and if virtualenv_deactivate does not call "pyenv uninstall ${version}"
+  # we get stale python2 in PATH that blocks NodeJS builds
+  export PATH=${PATH_WITHOUT_PYENV}
 }
 
 do_deepspeech_nodejs_build()

taskcluster/test-darwin-opt-base.tyml

@@ -25,6 +25,8 @@ then:
     "queue:route:notify.irc-channel.*",
     "generic-worker:cache:deepspeech-homebrew-bin",
     "generic-worker:cache:deepspeech-homebrew-cache",
+    "generic-worker:cache:deepspeech-homebrew-local",
+    "generic-worker:cache:deepspeech-macos-pyenv",
   ]
 
   payload:
@@ -57,6 +59,8 @@ then:
             (mkdir ../tc-workdir/ || rm -fr ../tc-workdir/*) && cd ../tc-workdir/ &&
             (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew/ homebrew/ || true) &&
             (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ homebrew.cache/ || true) &&
+            (mv $TASKCLUSTER_ORIG_TASKDIR/homebrew.local/ homebrew.local/ || true) &&
+            (mv $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ pyenv.cache/ || true) &&
             export TASKCLUSTER_TASK_DIR="$(pwd)" &&
             export TASKCLUSTER_TMP_DIR="$TASKCLUSTER_TASK_DIR/tmp" &&
             export LC_ALL=C &&
@@ -68,6 +72,8 @@ then:
             trap "export TASKCLUSTER_TASK_EXIT_CODE=$? &&
             mv $TASKCLUSTER_TASK_DIR/homebrew/ $TASKCLUSTER_ORIG_TASKDIR/homebrew/ &&
             mv $TASKCLUSTER_TASK_DIR/homebrew.cache/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.cache/ &&
+            mv $TASKCLUSTER_TASK_DIR/homebrew.local/ $TASKCLUSTER_ORIG_TASKDIR/homebrew.local/ &&
+            mv $TASKCLUSTER_TASK_DIR/pyenv.cache/ $TASKCLUSTER_ORIG_TASKDIR/pyenv.cache/ &&
             cd $TASKCLUSTER_TASK_DIR/../ && rm -fr tc-workdir/ && exit $TASKCLUSTER_TASK_EXIT_CODE" EXIT &&
             git clone --quiet ${event.head.repo.url} $TASKCLUSTER_TASK_DIR/DeepSpeech/ds/ &&
             cd $TASKCLUSTER_TASK_DIR/DeepSpeech/ds && git checkout --quiet ${event.head.sha} &&
@@ -80,6 +86,10 @@ then:
         directory: homebrew/
       - cacheName: deepspeech-homebrew-cache
         directory: homebrew.cache/
+      - cacheName: deepspeech-homebrew-local
+        directory: homebrew.local/
+      - cacheName: deepspeech-macos-pyenv
+        directory: pyenv.cache/
 
   metadata:
     name: ${build.metadata.name}