experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Deb and pip package cleanups; faster protobuf install

Browse Source 
Installing protobuf from binary, instead of building it from source.
ci docker build performance comparison:
Not installing protobuf from binary: 9m55.948s
Installing protobuf from binary:     4m18.457s

(Note the total docker build time does not show as large an improvement because the build-from-source process of pip install pandas offsets that performance gain).

Remove some unused deb packages.
Install pandas with pip, instead of apt-get. This gets rid of the inadvertent installation of old numpy and scipy versions that have caused nightly build failures, e.g., http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/105/console

Unfortunately, pandas doesn't offer whl files for Linux. So we can't install them from binary like what we do for numpy, scipy and sklearn. So this will add some build time due to building pandas from source.
Change: 123496073
This commit is contained in:
Shanqing Cai 2016-05-28 05:32:31 -08:00 committed by TensorFlower Gardener
parent fe497c064f
commit f768aa720f
5 changed files with 41 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tensorflow/tools/ci_build/Dockerfile.android
View File

@ -18,6 +18,7 @@ ENV BAZELRC /root/.bazelrc
# Install extra libraries for android sdk. # Install extra libraries for android sdk.
# (see http://stackoverflow.com/questions/33427893/can-not-run-android-sdk-build-tools-23-0-2-aapt) # (see http://stackoverflow.com/questions/33427893/can-not-run-android-sdk-build-tools-23-0-2-aapt)
RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \
python-numpy \
lib32stdc++6 \ lib32stdc++6 \
lib32z1 \ lib32z1 \
&& \ && \

8
tensorflow/tools/ci_build/install/install_deb_packages.sh
View File

@ -19,31 +19,23 @@ set -e
# Install dependencies from ubuntu deb repository. # Install dependencies from ubuntu deb repository.
apt-get update apt-get update
# gfortran, atlas, blas and lapack required by scipy pip install
apt-get install -y \ apt-get install -y \
autoconf \ autoconf \
automake \ automake \
bc \
build-essential \ build-essential \
cmake \ cmake \
curl \ curl \
ffmpeg \ ffmpeg \
git \ git \
gfortran \
libatlas-base-dev \
libblas-dev \
libcurl4-openssl-dev \ libcurl4-openssl-dev \
liblapack-dev \
libtool \ libtool \
openjdk-8-jdk \ openjdk-8-jdk \
openjdk-8-jre-headless \ openjdk-8-jre-headless \
pkg-config \ pkg-config \
python-dev \ python-dev \
python-pandas \
python-pip \ python-pip \
python-virtualenv \ python-virtualenv \
python3-dev \ python3-dev \
python3-pandas \
python3-pip \ python3-pip \
sudo \ sudo \
swig \ swig \

57
tensorflow/tools/ci_build/install/install_pip_packages.sh
View File

@ -21,33 +21,33 @@ set -e
# Use pip to install numpy to the latest version, instead of 1.8.2 through # Use pip to install numpy to the latest version, instead of 1.8.2 through
# apt-get # apt-get
wget -q https://pypi.python.org/packages/06/92/3c786303889e6246971ad4c48ac2b4e37a1b1c67c0dc2106dc85cb15c18e/numpy-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl#md5=6ffb66ff78c28c55bfa09a2ceee487df wget -q https://pypi.python.org/packages/17/f3/404bc85be67150663024d2bb5af654c7d16cf678077690dda27b91be14eb/numpy-1.8.2-cp27-cp27mu-manylinux1_x86_64.whl#md5=3ccf5c004fc99bd06dd443de80d622e6
mv numpy-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl \ mv numpy-1.8.2-cp27-cp27mu-manylinux1_x86_64.whl \
numpy-1.11.0-cp27-none-linux_x86_64.whl numpy-1.8.2-cp27-none-linux_x86_64.whl
pip install numpy-1.11.0-cp27-none-linux_x86_64.whl pip install numpy-1.8.2-cp27-none-linux_x86_64.whl
rm numpy-1.11.0-cp27-none-linux_x86_64.whl rm numpy-1.8.2-cp27-none-linux_x86_64.whl
wget -q https://pypi.python.org/packages/ea/ca/5e48a68be496e6f79c3c8d90f7c03ea09bbb154ea4511f5b3d6c825cefe5/numpy-1.11.0-cp34-cp34m-manylinux1_x86_64.whl#md5=08a002aeffa20354aa5045eadb549361 wget -q https://pypi.python.org/packages/33/7d/46d8905d39f462e0f6d1f38e1d165adc2939b9f91ca800e1cba8ef0c0f24/numpy-1.8.2-cp34-cp34m-manylinux1_x86_64.whl#md5=528b2b555d2b6979f10e444cacc04fc9
mv numpy-1.11.0-cp34-cp34m-manylinux1_x86_64.whl \ mv numpy-1.8.2-cp34-cp34m-manylinux1_x86_64.whl \
numpy-1.11.0-cp34-cp34m-linux_x86_64.whl numpy-1.8.2-cp34-none-linux_x86_64.whl
pip3 install numpy-1.11.0-cp34-cp34m-linux_x86_64.whl pip3 install numpy-1.8.2-cp34-none-linux_x86_64.whl
rm numpy-1.11.0-cp34-cp34m-linux_x86_64.whl rm numpy-1.8.2-cp34-none-linux_x86_64.whl
# Use pip to install scipy to get the latest version, instead of 0.13 through # Use pip to install scipy to get the latest version, instead of 0.13 through
# apt-get. # apt-get.
# pip install scipy==0.17.1 # pip install scipy==0.15.1
wget -q https://pypi.python.org/packages/8a/de/326cf31a5a3ba0c01c40cdd78f7140b0510ed80e6d5ec5b2ec173c72df03/scipy-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=8d0df61ceba78a2796f8d90fc979576f wget -q https://pypi.python.org/packages/00/0f/060ec52cb74dc8df1a7ef1a524173eb0bcd329110404869b392685cfc5c8/scipy-0.15.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=aaac02e6535742ab02f2075129890714
mv scipy-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl \ mv scipy-0.15.1-cp27-cp27mu-manylinux1_x86_64.whl \
scipy-0.17.1-cp27-none-linux_x86_64.whl scipy-0.15.1-cp27-none-linux_x86_64.whl
pip install scipy-0.17.1-cp27-none-linux_x86_64.whl pip install scipy-0.15.1-cp27-none-linux_x86_64.whl
rm scipy-0.17.1-cp27-none-linux_x86_64.whl rm scipy-0.15.1-cp27-none-linux_x86_64.whl
# pip3 install scipy==0.17.1 # pip3 install scipy==0.15.1
wget -q https://pypi.python.org/packages/eb/2e/76aff3b25dd06cab06622f82a4790ff5002ab686e940847bb2503b4b2122/scipy-0.17.1-cp34-cp34m-manylinux1_x86_64.whl#md5=bb39b9e1d16fa220967ad7edd39a8b28 wget -q https://pypi.python.org/packages/56/c5/e0d36aaf719aa02ee3da19151045912e240d145586612e53b5eaa706e1db/scipy-0.15.1-cp34-cp34m-manylinux1_x86_64.whl#md5=d5243b0f9d85f4f4cb62514c82af93d4
mv scipy-0.17.1-cp34-cp34m-manylinux1_x86_64.whl \ mv scipy-0.15.1-cp34-cp34m-manylinux1_x86_64.whl \
scipy-0.17.1-cp34-cp34m-linux_x86_64.whl scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
pip3 install scipy-0.17.1-cp34-cp34m-linux_x86_64.whl pip3 install scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
rm scipy-0.17.1-cp34-cp34m-linux_x86_64.whl rm scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
# pip install sklearn # pip install sklearn
wget -q https://pypi.python.org/packages/bf/80/06e77e5a682c46a3880ec487a5f9d910f5c8d919df9aca58052089687c7e/scikit_learn-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=337b91f502138ba7fd722803138f6dfd wget -q https://pypi.python.org/packages/bf/80/06e77e5a682c46a3880ec487a5f9d910f5c8d919df9aca58052089687c7e/scikit_learn-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=337b91f502138ba7fd722803138f6dfd
@ -63,6 +63,10 @@ mv scikit_learn-0.17.1-cp34-cp34m-manylinux1_x86_64.whl \
pip3 install scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl pip3 install scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
rm scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl rm scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
# pandas required by tf.learn/inflow
pip install pandas==0.18.1
pip3 install pandas==0.18.1
# Benchmark tests require the following: # Benchmark tests require the following:
pip install psutil pip install psutil
pip3 install psutil pip3 install psutil
@ -72,12 +76,3 @@ pip3 install py-cpuinfo
# pylint tests require the following: # pylint tests require the following:
pip install pylint pip install pylint
pip3 install pylint pip3 install pylint
# Remove packages in /usr/lib/python* that may interfere with packages in
# /usr/local/lib. These packages may get installed inadvertantly with packages
# such as apt-get python-pandas. Their older versions can mask the more recent
# versions installed above with pip and cause test failures.
rm -rf /usr/lib/python2.7/dist-packages/numpy \
/usr/lib/python2.7/dist-packages/scipy \
/usr/lib/python3/dist-packages/numpy \
/usr/lib/python3/dist-packages/scipy

27
tensorflow/tools/ci_build/install/install_proto3.sh
View File

@ -16,19 +16,18 @@
set -e set -e
# Select bazel version. # Install protobuf3.
# Select protobuf version.
PROTOBUF_VERSION="3.0.0-beta-2" PROTOBUF_VERSION="3.0.0-beta-2"
# Install protobuf3. PROTOBUF_URL="https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip"
mkdir /protobuf PROTOBUF_ZIP=$(basename "${PROTOBUF_URL}")
cd /protobuf UNZIP_DEST="google-protobuf"
curl -fSsL -O https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-cpp-$PROTOBUF_VERSION.tar.gz
tar zxf protobuf-cpp-$PROTOBUF_VERSION.tar.gz wget -q "${PROTOBUF_URL}"
cd protobuf-$PROTOBUF_VERSION unzip "${PROTOBUF_ZIP}" -d "${UNZIP_DEST}"
./autogen.sh cp "${UNZIP_DEST}/protoc" /usr/local/bin/
./configure
make rm -f "${PROTOBUF_ZIP}"
make install rm -rf "${UNZIP_DEST}"
make clean
ldconfig
cd /; rm -rf /protobuf

3
tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
View File

@ -34,8 +34,6 @@ make install
rm -f /usr/bin/swig rm -f /usr/bin/swig
ln -s /usr/local/bin/swig /usr/bin/swig ln -s /usr/local/bin/swig /usr/bin/swig
which swig
popd popd
rm -rf swig-3.0.8 rm -rf swig-3.0.8
@ -61,3 +59,4 @@ pip3.5 install --upgrade scikit-learn
# Install recent-enough version of wheel for Python 3.5 wheel builds # Install recent-enough version of wheel for Python 3.5 wheel builds
pip3.5 install wheel==0.29.0 pip3.5 install wheel==0.29.0
pip3.5 install --upgrade pandas==0.18.1