Don't re-install dependencies if they exist on build machines.

Change: 140044561
This commit is contained in:
A. Unique TensorFlower 2016-11-23 10:06:33 -08:00 committed by TensorFlower Gardener
parent 51ae55c4e7
commit bfce39bcd9
7 changed files with 226 additions and 89 deletions

View File

@ -18,13 +18,16 @@ set -e
sudo pip3 install auditwheel
# Install patchelf from source (it does not come with trusty package)
wget https://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.bz2
tar xfa patchelf-0.9.tar.bz2
cd patchelf-0.9
./configure --prefix=/usr/local
make
sudo make install
set +e
patchelf_location=$(which patchelf)
if [[ -z "$patchelf_location" ]]; then
set -e
# Install patchelf from source (it does not come with trusty package)
wget https://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.bz2
tar xfa patchelf-0.9.tar.bz2
cd patchelf-0.9
./configure --prefix=/usr/local
make
sudo make install
fi
cd ..

View File

@ -14,15 +14,24 @@
# limitations under the License.
# ==============================================================================
set -e
# Select bazel version.
BAZEL_VERSION="0.3.2"
set +e
local_bazel_ver=$(bazel version 2>&1 | grep -i label | awk '{print $3}')
if [[ "$local_bazel_ver" == "$BAZEL_VERSION" ]]; then
exit 0
fi
set -e
# Install bazel.
mkdir /bazel
mkdir -p /bazel
cd /bazel
curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
if [[ ! -f "bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" ]]; then
curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
fi
chmod +x /bazel/bazel-*.sh
/bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh

View File

@ -15,17 +15,32 @@
# ==============================================================================
set -e
ubuntu_version=$(cat /etc/issue | grep -i ubuntu | awk '{print $2}' | \
awk -F'.' '{print $1}')
# Install dependencies from ubuntu deb repository.
apt-get update
set +e
ffmpeg_location=$(which ffmpeg)
if [[ -z "$ffmpeg_location" && "$ubuntu_version" == "14" ]]; then
set -e
# specifically for trusty linked from ffmpeg.org
add-apt-repository -y ppa:mc3man/trusty-media
apt-get update
apt-get dist-upgrade -y
apt-get install -y ffmpeg libav-tools
else
set -e
apt-get install -y ffmpeg libav-tools
fi
apt-get install -y --no-install-recommends \
autoconf \
automake \
build-essential \
cmake \
curl \
ffmpeg \
git \
libcurl4-openssl-dev \
libtool \
@ -46,7 +61,7 @@ apt-get install -y --no-install-recommends \
zlib1g-dev
# Install ca-certificates, and update the certificate store.
apt-get install ca-certificates-java
apt-get install -y ca-certificates-java
update-ca-certificates -f
apt-get clean

View File

@ -15,8 +15,12 @@
# ==============================================================================
# Install Hadoop
set -e
HADOOP_VERSION="2.7.3"
wget -q http://www-us.apache.org/dist/hadoop/common/hadoop-2.7.3/hadoop-2.7.3.tar.gz
tar xzf hadoop-2.7.3.tar.gz -C /usr/local
ln -s /usr/local/hadoop-2.7.3 /usr/local/hadoop
set +e
if [[ ! -f /usr/local/hadoop-${HADOOP_VERSION}/bin/hadoop ]]; then
set -e
wget -q http://www-us.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
tar xzf hadoop-${HADOOP_VERSION}.tar.gz -C /usr/local
fi
ln -sf /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop

View File

@ -33,49 +33,111 @@ pip3 install --upgrade protobuf==3.0.0
# Remove obsolete version of six, which can sometimes confuse virtualenv.
rm -rf /usr/lib/python3/dist-packages/six*
set +e
# Use pip to install numpy to a modern version, instead of 1.8.2 that comes
# with apt-get in ubuntu:14.04.
wget -q https://pypi.python.org/packages/06/92/3c786303889e6246971ad4c48ac2b4e37a1b1c67c0dc2106dc85cb15c18e/numpy-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl#md5=6ffb66ff78c28c55bfa09a2ceee487df
mv numpy-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl \
numpy-1.11.0-cp27-none-linux_x86_64.whl
pip install numpy-1.11.0-cp27-none-linux_x86_64.whl
rm numpy-1.11.0-cp27-none-linux_x86_64.whl
NUMPY_VERSION="1.11.0"
numpy_ver_flat=$(echo $NUMPY_VERSION | sed 's/\.//g' | sed 's/^0*//g')
local_numpy_ver=$(python -c "import numpy; print(numpy.__version__)")
local_numpy_ver_flat=$(echo $local_numpy_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_numpy_ver_flat ]]; then
local_numpy_ver_flat=0
fi
if (( $local_numpy_ver_flat < $numpy_ver_flat )); then
set -e
wget -q https://pypi.python.org/packages/06/92/3c786303889e6246971ad4c48ac2b4e37a1b1c67c0dc2106dc85cb15c18e/numpy-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl#md5=6ffb66ff78c28c55bfa09a2ceee487df
mv numpy-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl \
numpy-1.11.0-cp27-none-linux_x86_64.whl
pip install numpy-1.11.0-cp27-none-linux_x86_64.whl
rm numpy-1.11.0-cp27-none-linux_x86_64.whl
fi
wget -q https://pypi.python.org/packages/ea/ca/5e48a68be496e6f79c3c8d90f7c03ea09bbb154ea4511f5b3d6c825cefe5/numpy-1.11.0-cp34-cp34m-manylinux1_x86_64.whl#md5=08a002aeffa20354aa5045eadb549361
mv numpy-1.11.0-cp34-cp34m-manylinux1_x86_64.whl \
numpy-1.11.0-cp34-none-linux_x86_64.whl
pip3 install numpy-1.11.0-cp34-none-linux_x86_64.whl
rm numpy-1.11.0-cp34-none-linux_x86_64.whl
set +e
local_numpy_ver=$(python3 -c "import numpy; print(numpy.__version__)")
local_numpy_ver_flat=$(echo $local_numpy_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_numpy_ver_flat ]]; then
local_numpy_ver_flat=0
fi
if (( $local_numpy_ver_flat < $numpy_ver_flat )); then
set -e
wget -q https://pypi.python.org/packages/ea/ca/5e48a68be496e6f79c3c8d90f7c03ea09bbb154ea4511f5b3d6c825cefe5/numpy-1.11.0-cp34-cp34m-manylinux1_x86_64.whl#md5=08a002aeffa20354aa5045eadb549361
mv numpy-1.11.0-cp34-cp34m-manylinux1_x86_64.whl \
numpy-1.11.0-cp34-none-linux_x86_64.whl
pip3 install numpy-1.11.0-cp34-none-linux_x86_64.whl
rm numpy-1.11.0-cp34-none-linux_x86_64.whl
fi
# Use pip to install scipy to get the latest version, instead of 0.13 through
# apt-get.
# pip install scipy==0.15.1
wget -q https://pypi.python.org/packages/00/0f/060ec52cb74dc8df1a7ef1a524173eb0bcd329110404869b392685cfc5c8/scipy-0.15.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=aaac02e6535742ab02f2075129890714
mv scipy-0.15.1-cp27-cp27mu-manylinux1_x86_64.whl \
scipy-0.15.1-cp27-none-linux_x86_64.whl
pip install scipy-0.15.1-cp27-none-linux_x86_64.whl
rm scipy-0.15.1-cp27-none-linux_x86_64.whl
set +e
SCIPY_VERSION="0.15.1"
scipy_ver_flat=$(echo $SCIPY_VERSION | sed 's/\.//g' | sed 's/^0*//g')
local_scipy_ver=$(python -c "import scipy; print(scipy.__version__)")
local_scipy_ver_flat=$(echo $local_scipy_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_scipy_ver_flat ]]; then
local_scipy_ver_flat=0
fi
if (( $local_scipy_ver_flat < $scipy_ver_flat )); then
set -e
wget -q https://pypi.python.org/packages/00/0f/060ec52cb74dc8df1a7ef1a524173eb0bcd329110404869b392685cfc5c8/scipy-0.15.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=aaac02e6535742ab02f2075129890714
mv scipy-0.15.1-cp27-cp27mu-manylinux1_x86_64.whl \
scipy-0.15.1-cp27-none-linux_x86_64.whl
pip install scipy-0.15.1-cp27-none-linux_x86_64.whl
rm scipy-0.15.1-cp27-none-linux_x86_64.whl
fi
# pip3 install scipy==0.15.1
wget -q https://pypi.python.org/packages/56/c5/e0d36aaf719aa02ee3da19151045912e240d145586612e53b5eaa706e1db/scipy-0.15.1-cp34-cp34m-manylinux1_x86_64.whl#md5=d5243b0f9d85f4f4cb62514c82af93d4
mv scipy-0.15.1-cp34-cp34m-manylinux1_x86_64.whl \
scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
pip3 install scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
rm scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
set +e
local_scipy_ver=$(python3 -c "import scipy; print(scipy.__version__)")
local_scipy_ver_flat=$(echo $local_scipy_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_scipy_ver_flat ]]; then
local_scipy_ver_flat=0
fi
if (( $local_scipy_ver_flat < $scipy_ver_flat )); then
set -e
wget -q https://pypi.python.org/packages/56/c5/e0d36aaf719aa02ee3da19151045912e240d145586612e53b5eaa706e1db/scipy-0.15.1-cp34-cp34m-manylinux1_x86_64.whl#md5=d5243b0f9d85f4f4cb62514c82af93d4
mv scipy-0.15.1-cp34-cp34m-manylinux1_x86_64.whl \
scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
pip3 install scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
rm scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
fi
# pip install sklearn
wget -q https://pypi.python.org/packages/bf/80/06e77e5a682c46a3880ec487a5f9d910f5c8d919df9aca58052089687c7e/scikit_learn-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=337b91f502138ba7fd722803138f6dfd
mv scikit_learn-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl \
scikit_learn-0.17.1-cp27-none-linux_x86_64.whl
pip install scikit_learn-0.17.1-cp27-none-linux_x86_64.whl
rm scikit_learn-0.17.1-cp27-none-linux_x86_64.whl
set +e
SKLEARN_VERSION="0.17.1"
sklearn_ver_flat=$(echo $SKLEARN_VERSION | sed 's/\.//g' | sed 's/^0*//g')
local_sklearn_ver=$(python -c "import sklearn; print(sklearn.__version__)")
local_sklearn_ver_flat=$(echo $local_sklearn_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_sklearn_ver_flat ]]; then
local_sklearn_ver_flat=0
fi
if (( $local_sklearn_ver_flat < $sklearn_ver_flat )); then
set -e
wget -q https://pypi.python.org/packages/bf/80/06e77e5a682c46a3880ec487a5f9d910f5c8d919df9aca58052089687c7e/scikit_learn-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=337b91f502138ba7fd722803138f6dfd
mv scikit_learn-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl \
scikit_learn-0.17.1-cp27-none-linux_x86_64.whl
pip install scikit_learn-0.17.1-cp27-none-linux_x86_64.whl
rm scikit_learn-0.17.1-cp27-none-linux_x86_64.whl
fi
# pip3 install scikit-learn
wget -q https://pypi.python.org/packages/7e/f1/1cc8a1ae2b4de89bff0981aee904ff05779c49a4c660fa38178f9772d3a7/scikit_learn-0.17.1-cp34-cp34m-manylinux1_x86_64.whl#md5=a722a7372b64ec9f7b49a2532d21372b
mv scikit_learn-0.17.1-cp34-cp34m-manylinux1_x86_64.whl \
scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
pip3 install scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
rm scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
set +e
local_sklearn_ver=$(python3 -c "import sklearn; print(sklearn.__version__)")
local_sklearn_ver_flat=$(echo $local_sklearn_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_sklearn_ver_flat ]]; then
local_sklearn_ver_flat=0
fi
if (( $local_sklearn_ver_flat < $sklearn_ver_flat )); then
set -e
wget -q https://pypi.python.org/packages/7e/f1/1cc8a1ae2b4de89bff0981aee904ff05779c49a4c660fa38178f9772d3a7/scikit_learn-0.17.1-cp34-cp34m-manylinux1_x86_64.whl#md5=a722a7372b64ec9f7b49a2532d21372b
mv scikit_learn-0.17.1-cp34-cp34m-manylinux1_x86_64.whl \
scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
pip3 install scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
rm scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
fi
set -e
# pandas required by tf.learn/inflow
pip install pandas==0.18.1

View File

@ -14,20 +14,26 @@
# limitations under the License.
# ==============================================================================
set -e
# Install protobuf3.
# Select protobuf version.
PROTOBUF_VERSION="3.1.0"
protobuf_ver_flat=$(echo $PROTOBUF_VERSION | sed 's/\.//g' | sed 's/^0*//g')
local_protobuf_ver=$(protoc --version | awk '{print $2}')
local_protobuf_ver_flat=$(echo $local_protobuf_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_protobuf_ver_flat ]]; then
local_protobuf_ver_flat=0
fi
if (( $local_protobuf_ver_flat < $protobuf_ver_flat )); then
set -e
PROTOBUF_URL="https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip"
PROTOBUF_ZIP=$(basename "${PROTOBUF_URL}")
UNZIP_DEST="google-protobuf"
PROTOBUF_URL="https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip"
PROTOBUF_ZIP=$(basename "${PROTOBUF_URL}")
UNZIP_DEST="google-protobuf"
wget -q "${PROTOBUF_URL}"
unzip "${PROTOBUF_ZIP}" -d "${UNZIP_DEST}"
cp "${UNZIP_DEST}/bin/protoc" /usr/local/bin/
wget -q "${PROTOBUF_URL}"
unzip "${PROTOBUF_ZIP}" -d "${UNZIP_DEST}"
cp "${UNZIP_DEST}/bin/protoc" /usr/local/bin/
rm -f "${PROTOBUF_ZIP}"
rm -rf "${UNZIP_DEST}"
rm -f "${PROTOBUF_ZIP}"
rm -rf "${UNZIP_DEST}"
fi

View File

@ -18,30 +18,34 @@
# TODO(cais): Remove this file once we upgrade to ubuntu:16.04 docker images for
# Python 3.5 builds.
set -e
# fkrull/deadsnakes is for Python3.5
add-apt-repository -y ppa:fkrull/deadsnakes
apt-get update
set +e
# Upgrade swig to 3.0.8
wget -q http://downloads.sourceforge.net/swig/swig-3.0.8.tar.gz
tar xzf swig-3.0.8.tar.gz
pushd /swig-3.0.8
apt-get install -y --no-install-recommends libpcre3-dev
./configure
make
make install
rm -f /usr/bin/swig
ln -s /usr/local/bin/swig /usr/bin/swig
popd
rm -rf swig-3.0.8
rm -f swig-3.0.8.tar.gz
SWIG_VERSION="3.0.8"
swig_ver_flat=$(echo $SWIG_VERSION | sed 's/\.//g' | sed 's/^0*//g')
local_swig_ver=$(swig -version | grep -i version | awk '{print $3}')
local_swig_ver_flat=$(echo $local_swig_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_swig_ver_flat ]]; then
local_swig_ver_flat=0
fi
if (( $local_swig_ver_flat < $swig_ver_flat )); then
set -e
wget -q http://downloads.sourceforge.net/swig/swig-3.0.8.tar.gz
tar xzf swig-3.0.8.tar.gz
pushd swig-3.0.8
apt-get install -y --no-install-recommends libpcre3-dev
./configure
make
make install
rm -f /usr/bin/swig
ln -s /usr/local/bin/swig /usr/bin/swig
popd
rm -rf swig-3.0.8 swig-3.0.8.tar.gz
fi
set -e
# Install Python 3.5 and dev library
apt-get install -y --no-install-recommends python3.5 libpython3.5-dev
@ -53,16 +57,38 @@ apt-get install -y --no-install-recommends python3.5 libpython3.5-dev
# install numpy for Python3.4 here so that the genrule will at least not
# complain about missing numpy. Once we upgrade to 16.04 for Python 3.5 builds,
# this will no longer be necessary.
wget -q https://bootstrap.pypa.io/get-pip.py
python3.4 get-pip.py
set +e
pip3_version=$(pip3 --version | grep "python 3.4")
if [[ -z $pip3_version ]]; then
set -e
wget -q https://bootstrap.pypa.io/get-pip.py
python3.4 get-pip.py
rm -f get-pip.py
fi
pip3 install --upgrade numpy==1.11.0
NUMPY_VERSION="1.11.0"
numpy_ver_flat=$(echo $NUMPY_VERSION | sed 's/\.//g' | sed 's/^0*//g')
local_numpy_ver=$(python3 -c "import numpy; print(numpy.__version__)")
local_numpy_ver_flat=$(echo $local_numpy_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_numpy_ver_flat ]]; then
local_numpy_ver_flat=0
fi
if (( $local_numpy_ver_flat < $numpy_ver_flat )); then
set -e
pip3 install --upgrade numpy==${NUMPY_VERSION}
fi
# Install pip3.5
wget -q https://bootstrap.pypa.io/get-pip.py
python3.5 get-pip.py
rm -f get-pip.py
set +e
pip35_version=$(pip3.5 --version | grep "python 3.5")
if [[ -z $pip35_version ]]; then
set -e
wget -q https://bootstrap.pypa.io/get-pip.py
python3.5 get-pip.py
rm -f get-pip.py
fi
set -e
# Install six.
pip3.5 install --upgrade six==1.10.0
@ -73,12 +99,24 @@ pip3.5 install --upgrade protobuf==3.0.0
rm -rf /usr/lib/python3/dist-packages/six*
# Install numpy, scipy and scikit-learn required by the builds
pip3.5 install --upgrade numpy==1.11.0
pip3.5 install --upgrade numpy
wget -q https://pypi.python.org/packages/91/f3/0052c245d53eb5f0e13b7215811e52af3791a8a7d31771605697c28466a0/scipy-0.17.1-cp35-cp35m-manylinux1_x86_64.whl#md5=8e77756904c81a6f79ed10e3abf0c544
pip3.5 install --upgrade scipy-0.17.1-cp35-cp35m-manylinux1_x86_64.whl
rm -f scipy-0.17.1-cp35-cp35m-manylinux1_x86_64.whl
set +e
SCIPY_VERSION="0.17.1"
scipy_ver_flat=$(echo $SCIPY_VERSION | sed 's/\.//g' | sed 's/^0*//g')
local_scipy_ver=$(python3.5 -c "import scipy; print(scipy.__version__)")
local_scipy_ver_flat=$(echo $local_scipy_ver | sed 's/\.//g' | sed 's/^0*//g')
if [[ -z $local_scipy_ver_flat ]]; then
local_scipy_ver_flat=0
fi
if (( $local_scipy_ver_flat < $scipy_ver_flat )); then
set -e
wget -q https://pypi.python.org/packages/91/f3/0052c245d53eb5f0e13b7215811e52af3791a8a7d31771605697c28466a0/scipy-0.17.1-cp35-cp35m-manylinux1_x86_64.whl#md5=8e77756904c81a6f79ed10e3abf0c544
pip3.5 install --upgrade scipy-0.17.1-cp35-cp35m-manylinux1_x86_64.whl
rm -f scipy-0.17.1-cp35-cp35m-manylinux1_x86_64.whl
fi
set -e
pip3.5 install --upgrade scikit-learn
# Install recent-enough version of wheel for Python 3.5 wheel builds