experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Simplify and pin common CI deps.

Browse Source 
This should make builds reproducible, enabling faster patch releases.

PiperOrigin-RevId: 337993165
Change-Id: I29bbe7d4026e56cd70fc6d933c9bb76f19faee63
This commit is contained in:
Mihai Maruseac 2020-10-19 21:30:02 -07:00 committed by TensorFlower Gardener
parent 36c8d2acd1
commit 38bc69da9c
3 changed files with 93 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
tensorflow/tools/ci_build/presubmit/macos/py2_cc/build.sh
View File

@ -1,68 +0,0 @@
#!/bin/bash
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# TODO(mihaimaruseac,hyey,ggadde): Convert to py3
set -e
# Error if we somehow forget to set the path to bazel_wrapper.py
set -u
BAZEL_WRAPPER_PATH=$1
set +u
# From this point on, logs can be publicly available
set -x
function setup_pip () {
install_pip2
python -m virtualenv tf_build_env --system-site-packages
source tf_build_env/bin/activate
install_macos_pip_deps
}
function run_build () {
# Run configure.
export TF_NEED_CUDA=0
export PYTHON_BIN_PATH=$(which python2)
yes "" | $PYTHON_BIN_PATH configure.py
tag_filters="-no_oss,-no_oss_py2,-gpu,-tpu,-benchmark-test,-nomac,-no_mac,-v1only"
# Get the default test targets for bazel.
source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh
"${BAZEL_WRAPPER_PATH}" \
test \
--build_tag_filters="${tag_filters}" \
--test_tag_filters="${tag_filters}" \
--action_env=PATH \
--remote_accept_cached=true \
--spawn_strategy=standalone \
--remote_local_fallback=false \
--remote_timeout=600 \
--strategy=Javac=standalone \
--strategy=Closure=standalone \
--genrule_strategy=standalone \
-- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/...
# Copy log to output to be available to GitHub
ls -la "$(bazel info output_base)/java.log"
cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/"
}
source tensorflow/tools/ci_build/release/common.sh
install_bazelisk
setup_pip
run_build

153
tensorflow/tools/ci_build/release/common.sh
View File

@ -103,50 +103,6 @@ function update_bazel_linux {
# LINT.ThenChange( # LINT.ThenChange(
# //tensorflow_estimator/google/kokoro/common.sh) # //tensorflow_estimator/google/kokoro/common.sh)
function install_pip2 {
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
sudo python2 get-pip.py
}
function install_pip_deps {
SUDO_CMD=""
PIP_CMD="pip"
while true; do
if [[ -z "${1}" ]]; then
break
fi
if [[ "$1" == "sudo" ]]; then
SUDO_CMD="sudo "
elif [[ "$1" == "pip"* ]]; then
PIP_CMD="$1"
fi
shift
done
# LINT.IfChange(ubuntu_pip_installations)
# TODO(aselle): Change all these to be --user instead of sudo.
${SUDO_CMD} ${PIP_CMD} install astunparse==1.6.3
${SUDO_CMD} ${PIP_CMD} install keras_preprocessing==1.1.0 --no-deps
"${PIP_CMD}" install numpy==1.16.0 --user
"${PIP_CMD}" install PyYAML==3.13 --user
${SUDO_CMD} ${PIP_CMD} install gast==0.3.3
${SUDO_CMD} ${PIP_CMD} install h5py==2.10.0
${SUDO_CMD} ${PIP_CMD} install six==1.12.0
${SUDO_CMD} ${PIP_CMD} install grpcio
${SUDO_CMD} ${PIP_CMD} install portpicker
${SUDO_CMD} ${PIP_CMD} install scipy
${SUDO_CMD} ${PIP_CMD} install scikit-learn
${SUDO_CMD} ${PIP_CMD} install typing_extensions
${SUDO_CMD} ${PIP_CMD} install --upgrade tb-nightly
${PIP_CMD} install --user --upgrade flatbuffers
${PIP_CMD} install --user --upgrade attrs
${PIP_CMD} install --user --upgrade tf-estimator-nightly
${PIP_CMD} install --user --upgrade "future>=0.17.1"
${PIP_CMD} install --user --upgrade wrapt
# LINT.ThenChange(:ubuntu_16_pip_installations)
}
function install_ubuntu_16_pip_deps { function install_ubuntu_16_pip_deps {
PIP_CMD="pip" PIP_CMD="pip"
@ -160,30 +116,42 @@ function install_ubuntu_16_pip_deps {
shift shift
done done
# LINT.IfChange(ubuntu_16_pip_installations) # LINT.IfChange(linux_pip_installations)
"${PIP_CMD}" install astunparse==1.6.3 --user # To have reproducible builds, these dependencies should be pinned always.
"${PIP_CMD}" install --user --upgrade attrs # Prefer pinning to the same version as in setup.py
"${PIP_CMD}" install --user --upgrade flatbuffers # First, upgrade pypi wheels
"${PIP_CMD}" install keras_preprocessing==1.1.0 --no-deps --user "${PIP_CMD}" install --user --upgrade setuptools pip wheel
"${PIP_CMD}" install numpy==1.16.0 --user # Now, install the deps, as listed in setup.py
"${PIP_CMD}" install --user --upgrade "future>=0.17.1" "${PIP_CMD}" install --user 'absl-py ~= 0.10'
"${PIP_CMD}" install gast==0.3.3 --user "${PIP_CMD}" install --user 'astunparse ~= 1.6.3'
"${PIP_CMD}" install h5py==2.10.0 --user "${PIP_CMD}" install --user 'flatbuffers ~= 1.12.0'
"${PIP_CMD}" install six==1.12.0 --user "${PIP_CMD}" install --user 'google_pasta ~= 0.2'
"${PIP_CMD}" install grpcio --user "${PIP_CMD}" install --user 'h5py ~= 2.10.0'
"${PIP_CMD}" install portpicker --user "${PIP_CMD}" install --user 'keras_preprocessing ~= 1.1.2'
"${PIP_CMD}" install scipy --user "${PIP_CMD}" install --user 'numpy ~= 1.19.2'
"${PIP_CMD}" install scikit-learn --user "${PIP_CMD}" install --user 'opt_einsum ~= 3.3.0'
"${PIP_CMD}" install typing_extensions --user "${PIP_CMD}" install --user 'protobuf ~= 3.13.0'
"${PIP_CMD}" install PyYAML==3.13 --user "${PIP_CMD}" install --user 'six ~= 1.15.0'
# b/156523241 "${PIP_CMD}" install --user 'termcolor ~= 1.1.0'
"${PIP_CMD}" install --force-reinstall --user --upgrade tf-estimator-nightly "${PIP_CMD}" install --user 'typing_extensions ~= 3.7.4'
"${PIP_CMD}" install --user --upgrade tb-nightly "${PIP_CMD}" install --user 'wheel ~= 0.35'
"${PIP_CMD}" install --user --upgrade wrapt "${PIP_CMD}" install --user 'wrapt ~= 1.12.1'
# LINT.ThenChange(:ubuntu_pip_installations) # We need to pin gast dependency exactly
"${PIP_CMD}" install --user 'gast == 0.3.3'
# Finally, install tensorboard and estimator
# Note that here we want the latest version that matches (b/156523241)
"${PIP_CMD}" install --user --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a'
"${PIP_CMD}" install --user --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0'
# Test dependencies
"${PIP_CMD}" install --user 'grpcio ~= 1.32.0'
"${PIP_CMD}" install --user 'portpicker ~= 1.3.1'
"${PIP_CMD}" install --user 'scipy ~= 1.5.2'
# LINT.ThenChange(:mac_pip_installations)
} }
function install_macos_pip_deps { function install_macos_pip_deps {
# TODO(mihaimaruseac): Remove need for sudo, then this can be merged with
# above (probably needs to convert to venv too).
SUDO_CMD="" SUDO_CMD=""
PIP_CMD="pip" PIP_CMD="pip"
@ -202,30 +170,37 @@ function install_macos_pip_deps {
shift shift
done done
# High Sierra pip for Python2.7 installs don't work as expected. # LINT.IfChange(mac_pip_installations)
if [[ "${PIP_CMD}" == "pip" ]]; then # To have reproducible builds, these dependencies should be pinned always.
PIP_CMD="python -m pip" # Prefer pinning to the same version as in setup.py
SUDO_CMD="sudo -H " # First, upgrade pypi wheels
fi "${PIP_CMD}" install --user --upgrade setuptools pip wheel
# Now, install the deps, as listed in setup.py
# TODO(aselle): Change all these to be --user instead of sudo. "${PIP_CMD}" install --user 'absl-py ~= 0.10'
${SUDO_CMD} ${PIP_CMD} install --upgrade setuptools==39.1.0 "${PIP_CMD}" install --user 'astunparse ~= 1.6.3'
${SUDO_CMD} ${PIP_CMD} install keras_preprocessing==1.1.0 --no-deps "${PIP_CMD}" install --user 'flatbuffers ~= 1.12.0'
${SUDO_CMD} ${PIP_CMD} install --upgrade mock portpicker scipy grpcio "${PIP_CMD}" install --user 'google_pasta ~= 0.2'
${SUDO_CMD} ${PIP_CMD} install six==1.12.0 "${PIP_CMD}" install --user 'h5py ~= 2.10.0'
${SUDO_CMD} ${PIP_CMD} install scikit-learn "${PIP_CMD}" install --user 'keras_preprocessing ~= 1.1.2'
${SUDO_CMD} ${PIP_CMD} install numpy==1.16.0 "${PIP_CMD}" install --user 'numpy ~= 1.19.2'
${SUDO_CMD} ${PIP_CMD} install gast==0.3.3 "${PIP_CMD}" install --user 'opt_einsum ~= 3.3.0'
${SUDO_CMD} ${PIP_CMD} install h5py==2.10.0 "${PIP_CMD}" install --user 'protobuf ~= 3.13.0'
${SUDO_CMD} ${PIP_CMD} install typing_extensions "${PIP_CMD}" install --user 'six ~= 1.15.0'
${SUDO_CMD} ${PIP_CMD} install --upgrade grpcio "${PIP_CMD}" install --user 'termcolor ~= 1.1.0'
${SUDO_CMD} ${PIP_CMD} install --upgrade tb-nightly "${PIP_CMD}" install --user 'typing_extensions ~= 3.7.4'
${PIP_CMD} install --user --upgrade flatbuffers "${PIP_CMD}" install --user 'wheel ~= 0.35'
${PIP_CMD} install --user --upgrade attrs "${PIP_CMD}" install --user 'wrapt ~= 1.12.1'
# b/156523241 # We need to pin gast dependency exactly
${PIP_CMD} install --force-reinstall --user --upgrade tf-estimator-nightly "${PIP_CMD}" install --user 'gast == 0.3.3'
${PIP_CMD} install --user --upgrade wrapt # Finally, install tensorboard and estimator
${PIP_CMD} install --user --upgrade "future>=0.17.1" # Note that here we want the latest version that matches (b/156523241)
"${PIP_CMD}" install --user --upgrade --force-reinstall 'tb-nightly ~= 2.4.0.a'
"${PIP_CMD}" install --user --upgrade --force-reinstall 'tensorflow_estimator ~= 2.3.0'
# Test dependencies
"${PIP_CMD}" install --user 'grpcio ~= 1.32.0'
"${PIP_CMD}" install --user 'portpicker ~= 1.3.1'
"${PIP_CMD}" install --user 'scipy ~= 1.5.2'
# LINT.ThenChange(:linux_pip_installations)
} }
function maybe_skip_v1 { function maybe_skip_v1 {

64
tensorflow/tools/ci_build/release/common_win.bat
View File

@ -21,43 +21,37 @@ IF NOT DEFINED PYTHON_DIRECTORY (
SET PYTHON_DIRECTORY=Python36 SET PYTHON_DIRECTORY=Python36
) )
SET PY_EXE=C:\%PYTHON_DIRECTORY%\python.exe SET PY_EXE=C:\%PYTHON_DIRECTORY%\python.exe
SET PIP_EXE=C:\%PYTHON_DIRECTORY%\Scripts\pip.exe
SET PATH=%PATH%;C:\%PYTHON_DIRECTORY% SET PATH=%PATH%;C:\%PYTHON_DIRECTORY%
@REM TODO(amitpatankar): Make an image with these packages and remove this. @REM To have reproducible builds, these dependencies should be pinned always.
@REM Prefer pinning to the same version as in setup.py
%PIP_EXE% install flatbuffers --upgrade --no-deps @REM First, upgrade pypi wheels
%PIP_EXE% install setuptools --upgrade %PY_EXE% -m pip install --upgrade setuptools pip wheel
%PIP_EXE% install future>=0.17.1 --no-deps @REM Now, install the deps, as listed in setup.py
%PIP_EXE% install --ignore-installed --force-reinstall --upgrade tf-estimator-nightly --no-deps %PY_EXE% -m pip install "absl-py ~= 0.10"
%PIP_EXE% install tb-nightly --no-deps %PY_EXE% -m pip install "astunparse ~= 1.6.3"
%PIP_EXE% install numpy==1.16.0 --upgrade --no-deps %PY_EXE% -m pip install "flatbuffers ~= 1.12.0"
%PIP_EXE% install opt_einsum --upgrade %PY_EXE% -m pip install "google_pasta ~= 0.2"
%PIP_EXE% install pandas --upgrade --no-deps %PY_EXE% -m pip install "h5py ~= 2.10.0"
%PIP_EXE% install protobuf --upgrade --no-deps %PY_EXE% -m pip install "keras_preprocessing ~= 1.1.2"
%PIP_EXE% install keras_preprocessing==1.1.0 --upgrade --no-deps %PY_EXE% -m pip install "numpy ~= 1.19.2"
%PIP_EXE% install wrapt --upgrade --no-deps %PY_EXE% -m pip install "opt_einsum ~= 3.3.0"
%PIP_EXE% install absl-py==0.9.0 %PY_EXE% -m pip install "protobuf ~= 3.13.0"
%PY_EXE% -m pip install "six ~= 1.15.0"
IF "%PYTHON_DIRECTORY%"=="Python37" ( %PY_EXE% -m pip install "termcolor ~= 1.1.0"
%PIP_EXE% install colorama==0.3.9 %PY_EXE% -m pip install "typing_extensions ~= 3.7.4"
%PIP_EXE% install cycler==0.10.0 %PY_EXE% -m pip install "wheel ~= 0.35"
%PIP_EXE% install jedi==0.11.1 %PY_EXE% -m pip install "wrapt ~= 1.12.1"
%PIP_EXE% install oauth2client==4.1.2 @REM We need to pin gast dependency exactly
%PIP_EXE% install portpicker==1.2.0 %PY_EXE% -m pip install "gast == 0.3.3"
%PIP_EXE% install parso==0.1.1 @REM Finally, install tensorboard and estimator
%PIP_EXE% install protobuf==3.8.0 @REM Note that here we want the latest version that matches (b/156523241)
%PIP_EXE% install scikit-learn==0.19.2 %PY_EXE% -m pip install --upgrade --force-reinstall "tb-nightly ~= 2.4.0.a"
%PIP_EXE% install scipy==1.1.0 %PY_EXE% -m pip install --upgrade --force-reinstall "tensorflow_estimator ~= 2.3.0"
%PIP_EXE% install termcolor==1.1.0 @REM Test dependencies
) %PY_EXE% -m pip install "grpcio ~= 1.32.0"
%PY_EXE% -m pip install "portpicker ~= 1.3.1"
@REM TODO(amitpatankar): this is just a quick fix so that windows build doesn't %PY_EXE% -m pip install "scipy ~= 1.5.2"
@REM break with gast upgrade to 0.3.3. Need to figure out the right way to
@REM handle this case.
%PIP_EXE% install gast==0.3.3
%PIP_EXE% install astunparse==1.6.3
%PIP_EXE% install typing_extensions
:: Set cuda related environment variables. If we are not using CUDA, these are not used. :: Set cuda related environment variables. If we are not using CUDA, these are not used.
IF NOT DEFINED TF_CUDA_VERSION ( IF NOT DEFINED TF_CUDA_VERSION (