experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert MacOS builds to use virtualenvs for Python.

Browse Source 
PiperOrigin-RevId: 353750028
Change-Id: Ibfacf4f599450b3b228ffd9d850e2fb5c50b798f
This commit is contained in:
Amit Patankar 2021-01-25 15:59:46 -08:00 committed by TensorFlower Gardener
parent ff2377f03e
commit 002669bb99
11 changed files with 49 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tensorflow/tools/ci_build/nightly_release/macos/cpu_py36.sh
View File

@ -23,9 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
install_macos_pip_deps sudo python3.6 # Set up and install MacOS pip dependencies.
setup_venv_macos python3.6
sudo pip3.6 install 'twine ~= 3.2.0'
# For python3 path on Mac # For python3 path on Mac
export PATH=$PATH:/usr/local/bin export PATH=$PATH:/usr/local/bin

5
tensorflow/tools/ci_build/nightly_release/macos/cpu_py37.sh
View File

@ -23,9 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
install_macos_pip_deps sudo python3.7 # Set up and install MacOS pip dependencies.
setup_venv_macos python3.7
sudo pip3.7 install 'twine ~= 3.2.0'
# For python3 path on Mac # For python3 path on Mac
export PATH=$PATH:/usr/local/bin export PATH=$PATH:/usr/local/bin

5
tensorflow/tools/ci_build/nightly_release/macos/cpu_py38.sh
View File

@ -23,9 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
install_macos_pip_deps sudo python3.8 # Set up and install MacOS pip dependencies.
setup_venv_macos python3.8
sudo pip3.8 install 'twine ~= 3.2.0'
# For python3 path on Mac # For python3 path on Mac
export PATH=$PATH:/usr/local/bin export PATH=$PATH:/usr/local/bin

9
tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh
View File

@ -24,12 +24,6 @@ set +u
# From this point on, logs can be publicly available # From this point on, logs can be publicly available
set -x set -x
function setup_pip () {
python3.7 -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
install_macos_pip_deps virtualenv
}
function run_build () { function run_build () {
# Run configure. # Run configure.
export TF_NEED_CUDA=0 export TF_NEED_CUDA=0
@ -62,5 +56,6 @@ function run_build () {
source tensorflow/tools/ci_build/release/common.sh source tensorflow/tools/ci_build/release/common.sh
install_bazelisk install_bazelisk
setup_pip # Set up and install MacOS pip dependencies.
setup_venv_macos python3.7
run_build run_build

6
tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh
View File

@ -23,11 +23,9 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
export MACOSX_DEPLOYMENT_TARGET=10.10 export MACOSX_DEPLOYMENT_TARGET=10.10
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
python3.6 -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
# Install macos pip dependencies # Set up and install MacOS pip dependencies.
install_macos_pip_deps virtualenv setup_venv_macos python3.6
# Run configure. # Run configure.
export TF_NEED_CUDA=0 export TF_NEED_CUDA=0

4
tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh
View File

@ -23,8 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
# Install macos pip dependencies # Set up and install MacOS pip dependencies.
install_macos_pip_deps sudo pip3.6 setup_venv_macos python3.6
# Export required variables for running pip_new.sh # Export required variables for running pip_new.sh
export OS_TYPE="MACOS" export OS_TYPE="MACOS"

6
tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh
View File

@ -22,11 +22,9 @@ install_bazelisk
# Pick a more recent version of xcode # Pick a more recent version of xcode
export DEVELOPER_DIR=/Applications/Xcode_11.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_11.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
python3.7 -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
# Install macos pip dependencies # Set up and install MacOS pip dependencies.
install_macos_pip_deps virtualenv setup_venv_macos python3.6
# Run configure. # Run configure.
export TF_NEED_CUDA=0 export TF_NEED_CUDA=0

4
tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh
View File

@ -23,8 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
# Install macos pip dependencies # Set up and install MacOS pip dependencies.
install_macos_pip_deps sudo pip3.7 setup_venv_macos python3.7
# Export required variables for running pip_new.sh # Export required variables for running pip_new.sh
export OS_TYPE="MACOS" export OS_TYPE="MACOS"

6
tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh
View File

@ -23,11 +23,9 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
export MACOSX_DEPLOYMENT_TARGET=10.10 export MACOSX_DEPLOYMENT_TARGET=10.10
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
python3.8 -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
# Install macos pip dependencies # Set up and install MacOS pip dependencies.
install_macos_pip_deps virtualenv setup_venv_macos python3.8
# Run configure. # Run configure.
export TF_NEED_CUDA=0 export TF_NEED_CUDA=0

4
tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh
View File

@ -23,8 +23,8 @@ install_bazelisk
export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer
sudo xcode-select -s "${DEVELOPER_DIR}" sudo xcode-select -s "${DEVELOPER_DIR}"
# Install macos pip dependencies # Set up and install MacOS pip dependencies.
install_macos_pip_deps sudo pip3.8 setup_venv_macos python3.8
# Export required variables for running pip_new.sh # Export required variables for running pip_new.sh
export OS_TYPE="MACOS" export OS_TYPE="MACOS"

75
tensorflow/tools/ci_build/release/common.sh
View File

@ -205,68 +205,51 @@ function install_ubuntu_16_python_pip_deps {
} }
function install_macos_pip_deps { function install_macos_pip_deps {
# TODO(mihaimaruseac): Remove need for sudo, then this can be merged with
# above (probably needs to convert to venv too).
SUDO_CMD=""
PIP_CMD="pip"
IS_VIRTUALENV=false
USER_FLAG="--user"
while true; do PIP_CMD="python -m pip"
if [[ -z "${1}" ]]; then
break
fi
if [[ "$1" == "sudo" ]]; then
SUDO_CMD="sudo "
elif [[ "$1" == "pip3.7" ]]; then
PIP_CMD="python3.7 -m pip"
SUDO_CMD="sudo -H "
elif [[ "$1" == "pip"* ]]; then
PIP_CMD="$1"
elif [[ "$1" == "virtualenv" ]]; then
IS_VIRTUALENV=true
PIP_CMD="pip"
USER_FLAG=""
fi
shift
done
# LINT.IfChange(mac_pip_installations) # LINT.IfChange(mac_pip_installations)
# To have reproducible builds, these dependencies should be pinned always. # To have reproducible builds, these dependencies should be pinned always.
# Prefer pinning to the same version as in setup.py # Prefer pinning to the same version as in setup.py
# First, upgrade pypi wheels # First, upgrade pypi wheels
${PIP_CMD} install $USER_FLAG --upgrade setuptools pip wheel ${PIP_CMD} install --upgrade setuptools pip wheel
# Now, install the deps, as listed in setup.py # Now, install the deps, as listed in setup.py
${PIP_CMD} install $USER_FLAG 'absl-py ~= 0.10' ${PIP_CMD} install 'absl-py ~= 0.10'
${PIP_CMD} install $USER_FLAG 'astunparse ~= 1.6.3' ${PIP_CMD} install 'astunparse ~= 1.6.3'
${PIP_CMD} install $USER_FLAG 'flatbuffers ~= 1.12.0' ${PIP_CMD} install 'flatbuffers ~= 1.12.0'
${PIP_CMD} install $USER_FLAG 'google_pasta ~= 0.2' ${PIP_CMD} install 'google_pasta ~= 0.2'
${PIP_CMD} install $USER_FLAG 'h5py ~= 3.1.0' ${PIP_CMD} install 'h5py ~= 3.1.0'
${PIP_CMD} install $USER_FLAG 'keras_preprocessing ~= 1.1.2' ${PIP_CMD} install 'keras_preprocessing ~= 1.1.2'
${PIP_CMD} install $USER_FLAG 'numpy ~= 1.19.2' ${PIP_CMD} install 'numpy ~= 1.19.2'
${PIP_CMD} install $USER_FLAG 'opt_einsum ~= 3.3.0' ${PIP_CMD} install 'opt_einsum ~= 3.3.0'
${PIP_CMD} install $USER_FLAG 'protobuf >= 3.9.2' ${PIP_CMD} install 'protobuf >= 3.9.2'
${PIP_CMD} install $USER_FLAG 'six ~= 1.15.0' ${PIP_CMD} install 'six ~= 1.15.0'
${PIP_CMD} install $USER_FLAG 'termcolor ~= 1.1.0' ${PIP_CMD} install 'termcolor ~= 1.1.0'
${PIP_CMD} install $USER_FLAG 'typing_extensions ~= 3.7.4' ${PIP_CMD} install 'typing_extensions ~= 3.7.4'
${PIP_CMD} install $USER_FLAG 'wheel ~= 0.35' ${PIP_CMD} install 'wheel ~= 0.35'
${PIP_CMD} install $USER_FLAG 'wrapt ~= 1.12.1' ${PIP_CMD} install 'wrapt ~= 1.12.1'
# We need to pin gast dependency exactly # We need to pin gast dependency exactly
${PIP_CMD} install $USER_FLAG 'gast == 0.4.0' ${PIP_CMD} install 'gast == 0.4.0'
# Finally, install tensorboard and estimator # Finally, install tensorboard and estimator
# Note that here we want the latest version that matches (b/156523241) # Note that here we want the latest version that matches (b/156523241)
${PIP_CMD} install $USER_FLAG --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a' ${PIP_CMD} install --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a'
${PIP_CMD} install $USER_FLAG --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0' ${PIP_CMD} install --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0'
# Test dependencies # Test dependencies
${PIP_CMD} install $USER_FLAG 'grpcio ~= 1.34.0' ${PIP_CMD} install 'grpcio ~= 1.34.0'
${PIP_CMD} install $USER_FLAG 'portpicker ~= 1.3.1' ${PIP_CMD} install 'portpicker ~= 1.3.1'
${PIP_CMD} install $USER_FLAG 'scipy ~= 1.5.2' ${PIP_CMD} install 'scipy ~= 1.5.2'
${PIP_CMD} install $USER_FLAG --upgrade certifi ${PIP_CMD} install --upgrade certifi
# LINT.ThenChange(:linux_pip_installations_orig) # LINT.ThenChange(:linux_pip_installations_orig)
# LINT.ThenChange(:linux_pip_installations) # LINT.ThenChange(:linux_pip_installations)
} }
function setup_venv_macos () {
# First argument needs to be the python executable.
${1} -m virtualenv tf_build_env
source tf_build_env/bin/activate
install_macos_pip_deps
}
function maybe_skip_v1 { function maybe_skip_v1 {
# If we are building with v2 by default, skip tests with v1only tag. # If we are building with v2 by default, skip tests with v1only tag.