diff --git a/tensorflow/tools/ci_build/nightly_release/macos/cpu_py36.sh b/tensorflow/tools/ci_build/nightly_release/macos/cpu_py36.sh index 201db7de981..8250e107d46 100644 --- a/tensorflow/tools/ci_build/nightly_release/macos/cpu_py36.sh +++ b/tensorflow/tools/ci_build/nightly_release/macos/cpu_py36.sh @@ -23,9 +23,8 @@ install_bazelisk export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer sudo xcode-select -s "${DEVELOPER_DIR}" -install_macos_pip_deps sudo python3.6 - -sudo pip3.6 install 'twine ~= 3.2.0' +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.6 # For python3 path on Mac export PATH=$PATH:/usr/local/bin diff --git a/tensorflow/tools/ci_build/nightly_release/macos/cpu_py37.sh b/tensorflow/tools/ci_build/nightly_release/macos/cpu_py37.sh index 7d348c75e16..df762f78de7 100644 --- a/tensorflow/tools/ci_build/nightly_release/macos/cpu_py37.sh +++ b/tensorflow/tools/ci_build/nightly_release/macos/cpu_py37.sh @@ -23,9 +23,8 @@ install_bazelisk export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer sudo xcode-select -s "${DEVELOPER_DIR}" -install_macos_pip_deps sudo python3.7 - -sudo pip3.7 install 'twine ~= 3.2.0' +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.7 # For python3 path on Mac export PATH=$PATH:/usr/local/bin diff --git a/tensorflow/tools/ci_build/nightly_release/macos/cpu_py38.sh b/tensorflow/tools/ci_build/nightly_release/macos/cpu_py38.sh index 41a46762914..31194286eb8 100644 --- a/tensorflow/tools/ci_build/nightly_release/macos/cpu_py38.sh +++ b/tensorflow/tools/ci_build/nightly_release/macos/cpu_py38.sh @@ -23,9 +23,8 @@ install_bazelisk export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer sudo xcode-select -s "${DEVELOPER_DIR}" -install_macos_pip_deps sudo python3.8 - -sudo pip3.8 install 'twine ~= 3.2.0' +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.8 # For python3 path on Mac export PATH=$PATH:/usr/local/bin diff --git a/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh b/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh index 7c55b43d61c..e5f92dbcb69 100644 --- a/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh +++ b/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh @@ -24,12 +24,6 @@ set +u # From this point on, logs can be publicly available set -x -function setup_pip () { - python3.7 -m virtualenv tf_build_env --system-site-packages - source tf_build_env/bin/activate - install_macos_pip_deps virtualenv -} - function run_build () { # Run configure. export TF_NEED_CUDA=0 @@ -62,5 +56,6 @@ function run_build () { source tensorflow/tools/ci_build/release/common.sh install_bazelisk -setup_pip +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.7 run_build diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh index 3f36d2c3220..a5f3a3e7f2f 100644 --- a/tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py36_nonpip.sh @@ -23,11 +23,9 @@ install_bazelisk export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export MACOSX_DEPLOYMENT_TARGET=10.10 sudo xcode-select -s "${DEVELOPER_DIR}" -python3.6 -m virtualenv tf_build_env --system-site-packages -source tf_build_env/bin/activate -# Install macos pip dependencies -install_macos_pip_deps virtualenv +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.6 # Run configure. export TF_NEED_CUDA=0 diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh index 375a8c705fa..ae5b3ad56d8 100644 --- a/tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py36_pip.sh @@ -23,8 +23,8 @@ install_bazelisk export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer sudo xcode-select -s "${DEVELOPER_DIR}" -# Install macos pip dependencies -install_macos_pip_deps sudo pip3.6 +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.6 # Export required variables for running pip_new.sh export OS_TYPE="MACOS" diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh index b3cddd0c109..74abdbe75bd 100644 --- a/tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py37_nonpip.sh @@ -22,11 +22,9 @@ install_bazelisk # Pick a more recent version of xcode export DEVELOPER_DIR=/Applications/Xcode_11.3.app/Contents/Developer sudo xcode-select -s "${DEVELOPER_DIR}" -python3.7 -m virtualenv tf_build_env --system-site-packages -source tf_build_env/bin/activate -# Install macos pip dependencies -install_macos_pip_deps virtualenv +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.6 # Run configure. export TF_NEED_CUDA=0 diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh index ea6779be698..886c14a21da 100644 --- a/tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py37_pip.sh @@ -23,8 +23,8 @@ install_bazelisk export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer sudo xcode-select -s "${DEVELOPER_DIR}" -# Install macos pip dependencies -install_macos_pip_deps sudo pip3.7 +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.7 # Export required variables for running pip_new.sh export OS_TYPE="MACOS" diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh index 70d742b6bf7..fd96e5588f6 100644 --- a/tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py38_nonpip.sh @@ -23,11 +23,9 @@ install_bazelisk export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer export MACOSX_DEPLOYMENT_TARGET=10.10 sudo xcode-select -s "${DEVELOPER_DIR}" -python3.8 -m virtualenv tf_build_env --system-site-packages -source tf_build_env/bin/activate -# Install macos pip dependencies -install_macos_pip_deps virtualenv +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.8 # Run configure. export TF_NEED_CUDA=0 diff --git a/tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh b/tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh index f0ef8e89766..288dfe98aa7 100644 --- a/tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh +++ b/tensorflow/tools/ci_build/rel/macos/cpu_py38_pip.sh @@ -23,8 +23,8 @@ install_bazelisk export DEVELOPER_DIR=/Applications/Xcode_10.3.app/Contents/Developer sudo xcode-select -s "${DEVELOPER_DIR}" -# Install macos pip dependencies -install_macos_pip_deps sudo pip3.8 +# Set up and install MacOS pip dependencies. +setup_venv_macos python3.8 # Export required variables for running pip_new.sh export OS_TYPE="MACOS" diff --git a/tensorflow/tools/ci_build/release/common.sh b/tensorflow/tools/ci_build/release/common.sh index 88fc1d385aa..8d16f3a9f46 100644 --- a/tensorflow/tools/ci_build/release/common.sh +++ b/tensorflow/tools/ci_build/release/common.sh @@ -205,68 +205,51 @@ function install_ubuntu_16_python_pip_deps { } function install_macos_pip_deps { - # TODO(mihaimaruseac): Remove need for sudo, then this can be merged with - # above (probably needs to convert to venv too). - SUDO_CMD="" - PIP_CMD="pip" - IS_VIRTUALENV=false - USER_FLAG="--user" - while true; do - if [[ -z "${1}" ]]; then - break - fi - if [[ "$1" == "sudo" ]]; then - SUDO_CMD="sudo " - elif [[ "$1" == "pip3.7" ]]; then - PIP_CMD="python3.7 -m pip" - SUDO_CMD="sudo -H " - elif [[ "$1" == "pip"* ]]; then - PIP_CMD="$1" - elif [[ "$1" == "virtualenv" ]]; then - IS_VIRTUALENV=true - PIP_CMD="pip" - USER_FLAG="" - fi - shift - done + PIP_CMD="python -m pip" # LINT.IfChange(mac_pip_installations) # To have reproducible builds, these dependencies should be pinned always. # Prefer pinning to the same version as in setup.py # First, upgrade pypi wheels - ${PIP_CMD} install $USER_FLAG --upgrade setuptools pip wheel + ${PIP_CMD} install --upgrade setuptools pip wheel # Now, install the deps, as listed in setup.py - ${PIP_CMD} install $USER_FLAG 'absl-py ~= 0.10' - ${PIP_CMD} install $USER_FLAG 'astunparse ~= 1.6.3' - ${PIP_CMD} install $USER_FLAG 'flatbuffers ~= 1.12.0' - ${PIP_CMD} install $USER_FLAG 'google_pasta ~= 0.2' - ${PIP_CMD} install $USER_FLAG 'h5py ~= 3.1.0' - ${PIP_CMD} install $USER_FLAG 'keras_preprocessing ~= 1.1.2' - ${PIP_CMD} install $USER_FLAG 'numpy ~= 1.19.2' - ${PIP_CMD} install $USER_FLAG 'opt_einsum ~= 3.3.0' - ${PIP_CMD} install $USER_FLAG 'protobuf >= 3.9.2' - ${PIP_CMD} install $USER_FLAG 'six ~= 1.15.0' - ${PIP_CMD} install $USER_FLAG 'termcolor ~= 1.1.0' - ${PIP_CMD} install $USER_FLAG 'typing_extensions ~= 3.7.4' - ${PIP_CMD} install $USER_FLAG 'wheel ~= 0.35' - ${PIP_CMD} install $USER_FLAG 'wrapt ~= 1.12.1' + ${PIP_CMD} install 'absl-py ~= 0.10' + ${PIP_CMD} install 'astunparse ~= 1.6.3' + ${PIP_CMD} install 'flatbuffers ~= 1.12.0' + ${PIP_CMD} install 'google_pasta ~= 0.2' + ${PIP_CMD} install 'h5py ~= 3.1.0' + ${PIP_CMD} install 'keras_preprocessing ~= 1.1.2' + ${PIP_CMD} install 'numpy ~= 1.19.2' + ${PIP_CMD} install 'opt_einsum ~= 3.3.0' + ${PIP_CMD} install 'protobuf >= 3.9.2' + ${PIP_CMD} install 'six ~= 1.15.0' + ${PIP_CMD} install 'termcolor ~= 1.1.0' + ${PIP_CMD} install 'typing_extensions ~= 3.7.4' + ${PIP_CMD} install 'wheel ~= 0.35' + ${PIP_CMD} install 'wrapt ~= 1.12.1' # We need to pin gast dependency exactly - ${PIP_CMD} install $USER_FLAG 'gast == 0.4.0' + ${PIP_CMD} install 'gast == 0.4.0' # Finally, install tensorboard and estimator # Note that here we want the latest version that matches (b/156523241) - ${PIP_CMD} install $USER_FLAG --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a' - ${PIP_CMD} install $USER_FLAG --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0' + ${PIP_CMD} install --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a' + ${PIP_CMD} install --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0' # Test dependencies - ${PIP_CMD} install $USER_FLAG 'grpcio ~= 1.34.0' - ${PIP_CMD} install $USER_FLAG 'portpicker ~= 1.3.1' - ${PIP_CMD} install $USER_FLAG 'scipy ~= 1.5.2' - ${PIP_CMD} install $USER_FLAG --upgrade certifi + ${PIP_CMD} install 'grpcio ~= 1.34.0' + ${PIP_CMD} install 'portpicker ~= 1.3.1' + ${PIP_CMD} install 'scipy ~= 1.5.2' + ${PIP_CMD} install --upgrade certifi # LINT.ThenChange(:linux_pip_installations_orig) # LINT.ThenChange(:linux_pip_installations) } +function setup_venv_macos () { + # First argument needs to be the python executable. + ${1} -m virtualenv tf_build_env + source tf_build_env/bin/activate + install_macos_pip_deps +} function maybe_skip_v1 { # If we are building with v2 by default, skip tests with v1only tag.