experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert MacOS builds to use virtualenvs for Python.

Browse Source 
PiperOrigin-RevId: 353750028
Change-Id: Ibfacf4f599450b3b228ffd9d850e2fb5c50b798f
This commit is contained in:
Amit Patankar 2021-01-25 15:59:46 -08:00 committed by TensorFlower Gardener
parent ff2377f03e
commit 002669bb99
11 changed files with 49 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tensorflow/tools/ci_build/nightly_release/macos/cpu_py36.sh
View File

@ -23,9 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}"
install_macos_pip_deps sudo python3.6
sudo pip3.6 install 'twine ~= 3.2.0'
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.6
# For python3 path on Mac
export PATH=$PATH:/usr/local/bin

5
tensorflow/tools/ci_build/nightly_release/macos/cpu_py37.sh
View File

@ -23,9 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}"
install_macos_pip_deps sudo python3.7
sudo pip3.7 install 'twine ~= 3.2.0'
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.7
# For python3 path on Mac
export PATH=$PATH:/usr/local/bin

5
tensorflow/tools/ci_build/nightly_release/macos/cpu_py38.sh
View File

@ -23,9 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}"
install_macos_pip_deps sudo python3.8
sudo pip3.8 install 'twine ~= 3.2.0'
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.8
# For python3 path on Mac
export PATH=$PATH:/usr/local/bin

9
tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh
View File

@ -24,12 +24,6 @@ set +u
# From this point on, logs can be publicly available
set -x
function setup_pip () {
python3.7 -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
install_macos_pip_deps virtualenv
}
function run_build () {
# Run configure.
export TF_NEED_CUDA=0
@ -62,5 +56,6 @@ function run_build () {
source tensorflow/tools/ci_build/release/common.sh
install_bazelisk
setup_pip
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.7
run_build

6
tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh
View File

@ -23,11 +23,9 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
export MACOSX_DEPLOYMENT_TARGET=10.10
sudo xcode-select -s "${DEVELOPER_DIR}"
python3.6 -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
# Install macos pip dependencies
install_macos_pip_deps virtualenv
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.6
# Run configure.
export TF_NEED_CUDA=0

4
tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh
View File

@ -23,8 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}"
# Install macos pip dependencies
install_macos_pip_deps sudo pip3.6
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.6
# Export required variables for running pip_new.sh
export OS_TYPE="MACOS"

6
tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh
View File

@ -22,11 +22,9 @@ install_bazelisk
# Pick a more recent version of xcode
export DEVELOPER_DIR=/Applications/Xcode_11.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}"
python3.7 -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
# Install macos pip dependencies
install_macos_pip_deps virtualenv
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.6
# Run configure.
export TF_NEED_CUDA=0

4
tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh
View File

@ -23,8 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}"
# Install macos pip dependencies
install_macos_pip_deps sudo pip3.7
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.7
# Export required variables for running pip_new.sh
export OS_TYPE="MACOS"

6
tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh
View File

@ -23,11 +23,9 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
export MACOSX_DEPLOYMENT_TARGET=10.10
sudo xcode-select -s "${DEVELOPER_DIR}"
python3.8 -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
# Install macos pip dependencies
install_macos_pip_deps virtualenv
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.8
# Run configure.
export TF_NEED_CUDA=0

4
tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh
View File

@ -23,8 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}"
# Install macos pip dependencies
install_macos_pip_deps sudo pip3.8
# Set up and install MacOS pip dependencies.
setup_venv_macos python3.8
# Export required variables for running pip_new.sh
export OS_TYPE="MACOS"

75
tensorflow/tools/ci_build/release/common.sh
View File

@ -205,68 +205,51 @@ function install_ubuntu_16_python_pip_deps {
}
function install_macos_pip_deps {
# TODO(mihaimaruseac): Remove need for sudo, then this can be merged with
# above (probably needs to convert to venv too).
SUDO_CMD=""
PIP_CMD="pip"
IS_VIRTUALENV=false
USER_FLAG="--user"
while true; do
if [[ -z "${1}" ]]; then
break
fi
if [[ "$1" == "sudo" ]]; then
SUDO_CMD="sudo "
elif [[ "$1" == "pip3.7" ]]; then
PIP_CMD="python3.7 -m pip"
SUDO_CMD="sudo -H "
elif [[ "$1" == "pip"* ]]; then
PIP_CMD="$1"
elif [[ "$1" == "virtualenv" ]]; then
IS_VIRTUALENV=true
PIP_CMD="pip"
USER_FLAG=""
fi
shift
done
PIP_CMD="python -m pip"
# LINT.IfChange(mac_pip_installations)
# To have reproducible builds, these dependencies should be pinned always.
# Prefer pinning to the same version as in setup.py
# First, upgrade pypi wheels
${PIP_CMD} install $USER_FLAG --upgrade setuptools pip wheel
${PIP_CMD} install --upgrade setuptools pip wheel
# Now, install the deps, as listed in setup.py
${PIP_CMD} install $USER_FLAG 'absl-py ~= 0.10'
${PIP_CMD} install $USER_FLAG 'astunparse ~= 1.6.3'
${PIP_CMD} install $USER_FLAG 'flatbuffers ~= 1.12.0'
${PIP_CMD} install $USER_FLAG 'google_pasta ~= 0.2'
${PIP_CMD} install $USER_FLAG 'h5py ~= 3.1.0'
${PIP_CMD} install $USER_FLAG 'keras_preprocessing ~= 1.1.2'
${PIP_CMD} install $USER_FLAG 'numpy ~= 1.19.2'
${PIP_CMD} install $USER_FLAG 'opt_einsum ~= 3.3.0'
${PIP_CMD} install $USER_FLAG 'protobuf >= 3.9.2'
${PIP_CMD} install $USER_FLAG 'six ~= 1.15.0'
${PIP_CMD} install $USER_FLAG 'termcolor ~= 1.1.0'
${PIP_CMD} install $USER_FLAG 'typing_extensions ~= 3.7.4'
${PIP_CMD} install $USER_FLAG 'wheel ~= 0.35'
${PIP_CMD} install $USER_FLAG 'wrapt ~= 1.12.1'
${PIP_CMD} install 'absl-py ~= 0.10'
${PIP_CMD} install 'astunparse ~= 1.6.3'
${PIP_CMD} install 'flatbuffers ~= 1.12.0'
${PIP_CMD} install 'google_pasta ~= 0.2'
${PIP_CMD} install 'h5py ~= 3.1.0'
${PIP_CMD} install 'keras_preprocessing ~= 1.1.2'
${PIP_CMD} install 'numpy ~= 1.19.2'
${PIP_CMD} install 'opt_einsum ~= 3.3.0'
${PIP_CMD} install 'protobuf >= 3.9.2'
${PIP_CMD} install 'six ~= 1.15.0'
${PIP_CMD} install 'termcolor ~= 1.1.0'
${PIP_CMD} install 'typing_extensions ~= 3.7.4'
${PIP_CMD} install 'wheel ~= 0.35'
${PIP_CMD} install 'wrapt ~= 1.12.1'
# We need to pin gast dependency exactly
${PIP_CMD} install $USER_FLAG 'gast == 0.4.0'
${PIP_CMD} install 'gast == 0.4.0'
# Finally, install tensorboard and estimator
# Note that here we want the latest version that matches (b/156523241)
${PIP_CMD} install $USER_FLAG --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a'
${PIP_CMD} install $USER_FLAG --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0'
${PIP_CMD} install --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a'
${PIP_CMD} install --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0'
# Test dependencies
${PIP_CMD} install $USER_FLAG 'grpcio ~= 1.34.0'
${PIP_CMD} install $USER_FLAG 'portpicker ~= 1.3.1'
${PIP_CMD} install $USER_FLAG 'scipy ~= 1.5.2'
${PIP_CMD} install $USER_FLAG --upgrade certifi
${PIP_CMD} install 'grpcio ~= 1.34.0'
${PIP_CMD} install 'portpicker ~= 1.3.1'
${PIP_CMD} install 'scipy ~= 1.5.2'
${PIP_CMD} install --upgrade certifi
# LINT.ThenChange(:linux_pip_installations_orig)
# LINT.ThenChange(:linux_pip_installations)
}
function setup_venv_macos () {
# First argument needs to be the python executable.
${1} -m virtualenv tf_build_env
source tf_build_env/bin/activate
install_macos_pip_deps
}
function maybe_skip_v1 {
# If we are building with v2 by default, skip tests with v1only tag.