Simplify and pin common CI deps.
This should make builds reproducible, enabling faster patch releases. PiperOrigin-RevId: 337993165 Change-Id: I29bbe7d4026e56cd70fc6d933c9bb76f19faee63
This commit is contained in:
Mihai Maruseac
TensorFlower Gardener
parent
36c8d2acd1
commit
38bc69da9c
@ -1,68 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
# TODO(mihaimaruseac,hyey,ggadde): Convert to py3
|
||||
|
||||
set -e
|
||||
|
||||
# Error if we somehow forget to set the path to bazel_wrapper.py
|
||||
set -u
|
||||
BAZEL_WRAPPER_PATH=$1
|
||||
set +u
|
||||
|
||||
# From this point on, logs can be publicly available
|
||||
set -x
|
||||
|
||||
function setup_pip () {
|
||||
install_pip2
|
||||
python -m virtualenv tf_build_env --system-site-packages
|
||||
source tf_build_env/bin/activate
|
||||
install_macos_pip_deps
|
||||
}
|
||||
|
||||
function run_build () {
|
||||
# Run configure.
|
||||
export TF_NEED_CUDA=0
|
||||
export PYTHON_BIN_PATH=$(which python2)
|
||||
yes "" | $PYTHON_BIN_PATH configure.py
|
||||
tag_filters="-no_oss,-no_oss_py2,-gpu,-tpu,-benchmark-test,-nomac,-no_mac,-v1only"
|
||||
|
||||
# Get the default test targets for bazel.
|
||||
source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh
|
||||
|
||||
"${BAZEL_WRAPPER_PATH}" \
|
||||
test \
|
||||
--build_tag_filters="${tag_filters}" \
|
||||
--test_tag_filters="${tag_filters}" \
|
||||
--action_env=PATH \
|
||||
--remote_accept_cached=true \
|
||||
--spawn_strategy=standalone \
|
||||
--remote_local_fallback=false \
|
||||
--remote_timeout=600 \
|
||||
--strategy=Javac=standalone \
|
||||
--strategy=Closure=standalone \
|
||||
--genrule_strategy=standalone \
|
||||
-- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/...
|
||||
|
||||
# Copy log to output to be available to GitHub
|
||||
ls -la "$(bazel info output_base)/java.log"
|
||||
cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/"
|
||||
}
|
||||
|
||||
source tensorflow/tools/ci_build/release/common.sh
|
||||
install_bazelisk
|
||||
|
||||
setup_pip
|
||||
run_build
|
||||
@ -103,50 +103,6 @@ function update_bazel_linux {
|
||||
# LINT.ThenChange(
|
||||
# //tensorflow_estimator/google/kokoro/common.sh)
|
||||
|
||||
function install_pip2 {
|
||||
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
|
||||
sudo python2 get-pip.py
|
||||
}
|
||||
|
||||
function install_pip_deps {
|
||||
SUDO_CMD=""
|
||||
PIP_CMD="pip"
|
||||
|
||||
while true; do
|
||||
if [[ -z "${1}" ]]; then
|
||||
break
|
||||
fi
|
||||
if [[ "$1" == "sudo" ]]; then
|
||||
SUDO_CMD="sudo "
|
||||
elif [[ "$1" == "pip"* ]]; then
|
||||
PIP_CMD="$1"
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
# LINT.IfChange(ubuntu_pip_installations)
|
||||
# TODO(aselle): Change all these to be --user instead of sudo.
|
||||
${SUDO_CMD} ${PIP_CMD} install astunparse==1.6.3
|
||||
${SUDO_CMD} ${PIP_CMD} install keras_preprocessing==1.1.0 --no-deps
|
||||
"${PIP_CMD}" install numpy==1.16.0 --user
|
||||
"${PIP_CMD}" install PyYAML==3.13 --user
|
||||
${SUDO_CMD} ${PIP_CMD} install gast==0.3.3
|
||||
${SUDO_CMD} ${PIP_CMD} install h5py==2.10.0
|
||||
${SUDO_CMD} ${PIP_CMD} install six==1.12.0
|
||||
${SUDO_CMD} ${PIP_CMD} install grpcio
|
||||
${SUDO_CMD} ${PIP_CMD} install portpicker
|
||||
${SUDO_CMD} ${PIP_CMD} install scipy
|
||||
${SUDO_CMD} ${PIP_CMD} install scikit-learn
|
||||
${SUDO_CMD} ${PIP_CMD} install typing_extensions
|
||||
${SUDO_CMD} ${PIP_CMD} install --upgrade tb-nightly
|
||||
${PIP_CMD} install --user --upgrade flatbuffers
|
||||
${PIP_CMD} install --user --upgrade attrs
|
||||
${PIP_CMD} install --user --upgrade tf-estimator-nightly
|
||||
${PIP_CMD} install --user --upgrade "future>=0.17.1"
|
||||
${PIP_CMD} install --user --upgrade wrapt
|
||||
# LINT.ThenChange(:ubuntu_16_pip_installations)
|
||||
}
|
||||
|
||||
function install_ubuntu_16_pip_deps {
|
||||
PIP_CMD="pip"
|
||||
|
||||
@ -160,30 +116,42 @@ function install_ubuntu_16_pip_deps {
|
||||
shift
|
||||
done
|
||||
|
||||
# LINT.IfChange(ubuntu_16_pip_installations)
|
||||
"${PIP_CMD}" install astunparse==1.6.3 --user
|
||||
"${PIP_CMD}" install --user --upgrade attrs
|
||||
"${PIP_CMD}" install --user --upgrade flatbuffers
|
||||
"${PIP_CMD}" install keras_preprocessing==1.1.0 --no-deps --user
|
||||
"${PIP_CMD}" install numpy==1.16.0 --user
|
||||
"${PIP_CMD}" install --user --upgrade "future>=0.17.1"
|
||||
"${PIP_CMD}" install gast==0.3.3 --user
|
||||
"${PIP_CMD}" install h5py==2.10.0 --user
|
||||
"${PIP_CMD}" install six==1.12.0 --user
|
||||
"${PIP_CMD}" install grpcio --user
|
||||
"${PIP_CMD}" install portpicker --user
|
||||
"${PIP_CMD}" install scipy --user
|
||||
"${PIP_CMD}" install scikit-learn --user
|
||||
"${PIP_CMD}" install typing_extensions --user
|
||||
"${PIP_CMD}" install PyYAML==3.13 --user
|
||||
# b/156523241
|
||||
"${PIP_CMD}" install --force-reinstall --user --upgrade tf-estimator-nightly
|
||||
"${PIP_CMD}" install --user --upgrade tb-nightly
|
||||
"${PIP_CMD}" install --user --upgrade wrapt
|
||||
# LINT.ThenChange(:ubuntu_pip_installations)
|
||||
# LINT.IfChange(linux_pip_installations)
|
||||
# To have reproducible builds, these dependencies should be pinned always.
|
||||
# Prefer pinning to the same version as in setup.py
|
||||
# First, upgrade pypi wheels
|
||||
"${PIP_CMD}" install --user --upgrade setuptools pip wheel
|
||||
# Now, install the deps, as listed in setup.py
|
||||
"${PIP_CMD}" install --user 'absl-py ~= 0.10'
|
||||
"${PIP_CMD}" install --user 'astunparse ~= 1.6.3'
|
||||
"${PIP_CMD}" install --user 'flatbuffers ~= 1.12.0'
|
||||
"${PIP_CMD}" install --user 'google_pasta ~= 0.2'
|
||||
"${PIP_CMD}" install --user 'h5py ~= 2.10.0'
|
||||
"${PIP_CMD}" install --user 'keras_preprocessing ~= 1.1.2'
|
||||
"${PIP_CMD}" install --user 'numpy ~= 1.19.2'
|
||||
"${PIP_CMD}" install --user 'opt_einsum ~= 3.3.0'
|
||||
"${PIP_CMD}" install --user 'protobuf ~= 3.13.0'
|
||||
"${PIP_CMD}" install --user 'six ~= 1.15.0'
|
||||
"${PIP_CMD}" install --user 'termcolor ~= 1.1.0'
|
||||
"${PIP_CMD}" install --user 'typing_extensions ~= 3.7.4'
|
||||
"${PIP_CMD}" install --user 'wheel ~= 0.35'
|
||||
"${PIP_CMD}" install --user 'wrapt ~= 1.12.1'
|
||||
# We need to pin gast dependency exactly
|
||||
"${PIP_CMD}" install --user 'gast == 0.3.3'
|
||||
# Finally, install tensorboard and estimator
|
||||
# Note that here we want the latest version that matches (b/156523241)
|
||||
"${PIP_CMD}" install --user --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a'
|
||||
"${PIP_CMD}" install --user --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0'
|
||||
# Test dependencies
|
||||
"${PIP_CMD}" install --user 'grpcio ~= 1.32.0'
|
||||
"${PIP_CMD}" install --user 'portpicker ~= 1.3.1'
|
||||
"${PIP_CMD}" install --user 'scipy ~= 1.5.2'
|
||||
# LINT.ThenChange(:mac_pip_installations)
|
||||
}
|
||||
|
||||
function install_macos_pip_deps {
|
||||
# TODO(mihaimaruseac): Remove need for sudo, then this can be merged with
|
||||
# above (probably needs to convert to venv too).
|
||||
SUDO_CMD=""
|
||||
PIP_CMD="pip"
|
||||
|
||||
@ -202,30 +170,37 @@ function install_macos_pip_deps {
|
||||
shift
|
||||
done
|
||||
|
||||
# High Sierra pip for Python2.7 installs don't work as expected.
|
||||
if [[ "${PIP_CMD}" == "pip" ]]; then
|
||||
PIP_CMD="python -m pip"
|
||||
SUDO_CMD="sudo -H "
|
||||
fi
|
||||
|
||||
# TODO(aselle): Change all these to be --user instead of sudo.
|
||||
${SUDO_CMD} ${PIP_CMD} install --upgrade setuptools==39.1.0
|
||||
${SUDO_CMD} ${PIP_CMD} install keras_preprocessing==1.1.0 --no-deps
|
||||
${SUDO_CMD} ${PIP_CMD} install --upgrade mock portpicker scipy grpcio
|
||||
${SUDO_CMD} ${PIP_CMD} install six==1.12.0
|
||||
${SUDO_CMD} ${PIP_CMD} install scikit-learn
|
||||
${SUDO_CMD} ${PIP_CMD} install numpy==1.16.0
|
||||
${SUDO_CMD} ${PIP_CMD} install gast==0.3.3
|
||||
${SUDO_CMD} ${PIP_CMD} install h5py==2.10.0
|
||||
${SUDO_CMD} ${PIP_CMD} install typing_extensions
|
||||
${SUDO_CMD} ${PIP_CMD} install --upgrade grpcio
|
||||
${SUDO_CMD} ${PIP_CMD} install --upgrade tb-nightly
|
||||
${PIP_CMD} install --user --upgrade flatbuffers
|
||||
${PIP_CMD} install --user --upgrade attrs
|
||||
# b/156523241
|
||||
${PIP_CMD} install --force-reinstall --user --upgrade tf-estimator-nightly
|
||||
${PIP_CMD} install --user --upgrade wrapt
|
||||
${PIP_CMD} install --user --upgrade "future>=0.17.1"
|
||||
# LINT.IfChange(mac_pip_installations)
|
||||
# To have reproducible builds, these dependencies should be pinned always.
|
||||
# Prefer pinning to the same version as in setup.py
|
||||
# First, upgrade pypi wheels
|
||||
"${PIP_CMD}" install --user --upgrade setuptools pip wheel
|
||||
# Now, install the deps, as listed in setup.py
|
||||
"${PIP_CMD}" install --user 'absl-py ~= 0.10'
|
||||
"${PIP_CMD}" install --user 'astunparse ~= 1.6.3'
|
||||
"${PIP_CMD}" install --user 'flatbuffers ~= 1.12.0'
|
||||
"${PIP_CMD}" install --user 'google_pasta ~= 0.2'
|
||||
"${PIP_CMD}" install --user 'h5py ~= 2.10.0'
|
||||
"${PIP_CMD}" install --user 'keras_preprocessing ~= 1.1.2'
|
||||
"${PIP_CMD}" install --user 'numpy ~= 1.19.2'
|
||||
"${PIP_CMD}" install --user 'opt_einsum ~= 3.3.0'
|
||||
"${PIP_CMD}" install --user 'protobuf ~= 3.13.0'
|
||||
"${PIP_CMD}" install --user 'six ~= 1.15.0'
|
||||
"${PIP_CMD}" install --user 'termcolor ~= 1.1.0'
|
||||
"${PIP_CMD}" install --user 'typing_extensions ~= 3.7.4'
|
||||
"${PIP_CMD}" install --user 'wheel ~= 0.35'
|
||||
"${PIP_CMD}" install --user 'wrapt ~= 1.12.1'
|
||||
# We need to pin gast dependency exactly
|
||||
"${PIP_CMD}" install --user 'gast == 0.3.3'
|
||||
# Finally, install tensorboard and estimator
|
||||
# Note that here we want the latest version that matches (b/156523241)
|
||||
"${PIP_CMD}" install --user --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a'
|
||||
"${PIP_CMD}" install --user --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0'
|
||||
# Test dependencies
|
||||
"${PIP_CMD}" install --user 'grpcio ~= 1.32.0'
|
||||
"${PIP_CMD}" install --user 'portpicker ~= 1.3.1'
|
||||
"${PIP_CMD}" install --user 'scipy ~= 1.5.2'
|
||||
# LINT.ThenChange(:linux_pip_installations)
|
||||
}
|
||||
|
||||
function maybe_skip_v1 {
|
||||
|
||||
@ -21,43 +21,37 @@ IF NOT DEFINED PYTHON_DIRECTORY (
|
||||
SET PYTHON_DIRECTORY=Python36
|
||||
)
|
||||
SET PY_EXE=C:\%PYTHON_DIRECTORY%\python.exe
|
||||
SET PIP_EXE=C:\%PYTHON_DIRECTORY%\Scripts\pip.exe
|
||||
SET PATH=%PATH%;C:\%PYTHON_DIRECTORY%
|
||||
|
||||
@REM TODO(amitpatankar): Make an image with these packages and remove this.
|
||||
|
||||
%PIP_EXE% install flatbuffers --upgrade --no-deps
|
||||
%PIP_EXE% install setuptools --upgrade
|
||||
%PIP_EXE% install future>=0.17.1 --no-deps
|
||||
%PIP_EXE% install --ignore-installed --force-reinstall --upgrade tf-estimator-nightly --no-deps
|
||||
%PIP_EXE% install tb-nightly --no-deps
|
||||
%PIP_EXE% install numpy==1.16.0 --upgrade --no-deps
|
||||
%PIP_EXE% install opt_einsum --upgrade
|
||||
%PIP_EXE% install pandas --upgrade --no-deps
|
||||
%PIP_EXE% install protobuf --upgrade --no-deps
|
||||
%PIP_EXE% install keras_preprocessing==1.1.0 --upgrade --no-deps
|
||||
%PIP_EXE% install wrapt --upgrade --no-deps
|
||||
%PIP_EXE% install absl-py==0.9.0
|
||||
|
||||
IF "%PYTHON_DIRECTORY%"=="Python37" (
|
||||
%PIP_EXE% install colorama==0.3.9
|
||||
%PIP_EXE% install cycler==0.10.0
|
||||
%PIP_EXE% install jedi==0.11.1
|
||||
%PIP_EXE% install oauth2client==4.1.2
|
||||
%PIP_EXE% install portpicker==1.2.0
|
||||
%PIP_EXE% install parso==0.1.1
|
||||
%PIP_EXE% install protobuf==3.8.0
|
||||
%PIP_EXE% install scikit-learn==0.19.2
|
||||
%PIP_EXE% install scipy==1.1.0
|
||||
%PIP_EXE% install termcolor==1.1.0
|
||||
)
|
||||
|
||||
@REM TODO(amitpatankar): this is just a quick fix so that windows build doesn't
|
||||
@REM break with gast upgrade to 0.3.3. Need to figure out the right way to
|
||||
@REM handle this case.
|
||||
%PIP_EXE% install gast==0.3.3
|
||||
%PIP_EXE% install astunparse==1.6.3
|
||||
%PIP_EXE% install typing_extensions
|
||||
@REM To have reproducible builds, these dependencies should be pinned always.
|
||||
@REM Prefer pinning to the same version as in setup.py
|
||||
@REM First, upgrade pypi wheels
|
||||
%PY_EXE% -m pip install --upgrade setuptools pip wheel
|
||||
@REM Now, install the deps, as listed in setup.py
|
||||
%PY_EXE% -m pip install "absl-py ~= 0.10"
|
||||
%PY_EXE% -m pip install "astunparse ~= 1.6.3"
|
||||
%PY_EXE% -m pip install "flatbuffers ~= 1.12.0"
|
||||
%PY_EXE% -m pip install "google_pasta ~= 0.2"
|
||||
%PY_EXE% -m pip install "h5py ~= 2.10.0"
|
||||
%PY_EXE% -m pip install "keras_preprocessing ~= 1.1.2"
|
||||
%PY_EXE% -m pip install "numpy ~= 1.19.2"
|
||||
%PY_EXE% -m pip install "opt_einsum ~= 3.3.0"
|
||||
%PY_EXE% -m pip install "protobuf ~= 3.13.0"
|
||||
%PY_EXE% -m pip install "six ~= 1.15.0"
|
||||
%PY_EXE% -m pip install "termcolor ~= 1.1.0"
|
||||
%PY_EXE% -m pip install "typing_extensions ~= 3.7.4"
|
||||
%PY_EXE% -m pip install "wheel ~= 0.35"
|
||||
%PY_EXE% -m pip install "wrapt ~= 1.12.1"
|
||||
@REM We need to pin gast dependency exactly
|
||||
%PY_EXE% -m pip install "gast == 0.3.3"
|
||||
@REM Finally, install tensorboard and estimator
|
||||
@REM Note that here we want the latest version that matches (b/156523241)
|
||||
%PY_EXE% -m pip install --upgrade --force-reinstall "tb-nightly ~= 2.4.0.a"
|
||||
%PY_EXE% -m pip install --upgrade --force-reinstall "tensorflow_estimator ~= 2.3.0"
|
||||
@REM Test dependencies
|
||||
%PY_EXE% -m pip install "grpcio ~= 1.32.0"
|
||||
%PY_EXE% -m pip install "portpicker ~= 1.3.1"
|
||||
%PY_EXE% -m pip install "scipy ~= 1.5.2"
|
||||
|
||||
:: Set cuda related environment variables. If we are not using CUDA, these are not used.
|
||||
IF NOT DEFINED TF_CUDA_VERSION (
|
||||
|
||||
Loading…
Reference in New Issue
Block a user