Deb and pip package cleanups; faster protobuf install

Installing protobuf from binary, instead of building it from source.
ci docker build performance comparison:
Not installing protobuf from binary: 9m55.948s
Installing protobuf from binary:     4m18.457s

(Note the total docker build time does not show as large an improvement because the build-from-source process of pip install pandas offsets that performance gain).

Remove some unused deb packages.
Install pandas with pip, instead of apt-get. This gets rid of the inadvertent installation of old numpy and scipy versions that have caused nightly build failures, e.g., http://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_CONTAINER_TYPE=CPU,TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/105/console

Unfortunately, pandas doesn't offer whl files for Linux. So we can't install them from binary like what we do for numpy, scipy and sklearn. So this will add some build time due to building pandas from source.
Change: 123496073

tensorflow/tools/ci_build/Dockerfile.android

@@ -18,6 +18,7 @@ ENV BAZELRC /root/.bazelrc
 # Install extra libraries for android sdk.
 # (see http://stackoverflow.com/questions/33427893/can-not-run-android-sdk-build-tools-23-0-2-aapt)
 RUN apt-get update && apt-get install -y \
+        python-numpy \
         lib32stdc++6 \
         lib32z1 \
         && \

tensorflow/tools/ci_build/install/install_deb_packages.sh

@@ -19,31 +19,23 @@ set -e
 # Install dependencies from ubuntu deb repository.
 apt-get update
 
-# gfortran, atlas, blas and lapack required by scipy pip install
 apt-get install -y \
     autoconf \
     automake \
-    bc \
     build-essential \
     cmake \
     curl \
     ffmpeg \
     git \
-    gfortran \
-    libatlas-base-dev \
-    libblas-dev \
     libcurl4-openssl-dev \
-    liblapack-dev \
     libtool \
     openjdk-8-jdk \
     openjdk-8-jre-headless \
     pkg-config \
     python-dev \
-    python-pandas \
     python-pip \
     python-virtualenv \
     python3-dev \
-    python3-pandas \
     python3-pip \
     sudo \
     swig \

tensorflow/tools/ci_build/install/install_pip_packages.sh

@@ -21,33 +21,33 @@ set -e
 
 # Use pip to install numpy to the latest version, instead of 1.8.2 through
 # apt-get
-wget -q https://pypi.python.org/packages/06/92/3c786303889e6246971ad4c48ac2b4e37a1b1c67c0dc2106dc85cb15c18e/numpy-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl#md5=6ffb66ff78c28c55bfa09a2ceee487df
-mv numpy-1.11.0-cp27-cp27mu-manylinux1_x86_64.whl \
-   numpy-1.11.0-cp27-none-linux_x86_64.whl
-pip install numpy-1.11.0-cp27-none-linux_x86_64.whl
-rm numpy-1.11.0-cp27-none-linux_x86_64.whl
+wget -q https://pypi.python.org/packages/17/f3/404bc85be67150663024d2bb5af654c7d16cf678077690dda27b91be14eb/numpy-1.8.2-cp27-cp27mu-manylinux1_x86_64.whl#md5=3ccf5c004fc99bd06dd443de80d622e6
+mv numpy-1.8.2-cp27-cp27mu-manylinux1_x86_64.whl \
+   numpy-1.8.2-cp27-none-linux_x86_64.whl
+pip install numpy-1.8.2-cp27-none-linux_x86_64.whl
+rm numpy-1.8.2-cp27-none-linux_x86_64.whl
 
-wget -q https://pypi.python.org/packages/ea/ca/5e48a68be496e6f79c3c8d90f7c03ea09bbb154ea4511f5b3d6c825cefe5/numpy-1.11.0-cp34-cp34m-manylinux1_x86_64.whl#md5=08a002aeffa20354aa5045eadb549361
-mv numpy-1.11.0-cp34-cp34m-manylinux1_x86_64.whl \
-   numpy-1.11.0-cp34-cp34m-linux_x86_64.whl
-pip3 install numpy-1.11.0-cp34-cp34m-linux_x86_64.whl
-rm numpy-1.11.0-cp34-cp34m-linux_x86_64.whl
+wget -q https://pypi.python.org/packages/33/7d/46d8905d39f462e0f6d1f38e1d165adc2939b9f91ca800e1cba8ef0c0f24/numpy-1.8.2-cp34-cp34m-manylinux1_x86_64.whl#md5=528b2b555d2b6979f10e444cacc04fc9
+mv numpy-1.8.2-cp34-cp34m-manylinux1_x86_64.whl \
+   numpy-1.8.2-cp34-none-linux_x86_64.whl
+pip3 install numpy-1.8.2-cp34-none-linux_x86_64.whl
+rm numpy-1.8.2-cp34-none-linux_x86_64.whl
 
 # Use pip to install scipy to get the latest version, instead of 0.13 through
 # apt-get.
-# pip install scipy==0.17.1
-wget -q https://pypi.python.org/packages/8a/de/326cf31a5a3ba0c01c40cdd78f7140b0510ed80e6d5ec5b2ec173c72df03/scipy-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=8d0df61ceba78a2796f8d90fc979576f
-mv scipy-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl \
-   scipy-0.17.1-cp27-none-linux_x86_64.whl
-pip install scipy-0.17.1-cp27-none-linux_x86_64.whl
-rm scipy-0.17.1-cp27-none-linux_x86_64.whl
+# pip install scipy==0.15.1
+wget -q https://pypi.python.org/packages/00/0f/060ec52cb74dc8df1a7ef1a524173eb0bcd329110404869b392685cfc5c8/scipy-0.15.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=aaac02e6535742ab02f2075129890714
+mv scipy-0.15.1-cp27-cp27mu-manylinux1_x86_64.whl \
+   scipy-0.15.1-cp27-none-linux_x86_64.whl
+pip install scipy-0.15.1-cp27-none-linux_x86_64.whl
+rm scipy-0.15.1-cp27-none-linux_x86_64.whl
 
-# pip3 install scipy==0.17.1
-wget -q https://pypi.python.org/packages/eb/2e/76aff3b25dd06cab06622f82a4790ff5002ab686e940847bb2503b4b2122/scipy-0.17.1-cp34-cp34m-manylinux1_x86_64.whl#md5=bb39b9e1d16fa220967ad7edd39a8b28
-mv scipy-0.17.1-cp34-cp34m-manylinux1_x86_64.whl \
-   scipy-0.17.1-cp34-cp34m-linux_x86_64.whl
-pip3 install scipy-0.17.1-cp34-cp34m-linux_x86_64.whl
-rm scipy-0.17.1-cp34-cp34m-linux_x86_64.whl
+# pip3 install scipy==0.15.1
+wget -q https://pypi.python.org/packages/56/c5/e0d36aaf719aa02ee3da19151045912e240d145586612e53b5eaa706e1db/scipy-0.15.1-cp34-cp34m-manylinux1_x86_64.whl#md5=d5243b0f9d85f4f4cb62514c82af93d4
+mv scipy-0.15.1-cp34-cp34m-manylinux1_x86_64.whl \
+   scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
+pip3 install scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
+rm scipy-0.15.1-cp34-cp34m-linux_x86_64.whl
 
 # pip install sklearn
 wget -q https://pypi.python.org/packages/bf/80/06e77e5a682c46a3880ec487a5f9d910f5c8d919df9aca58052089687c7e/scikit_learn-0.17.1-cp27-cp27mu-manylinux1_x86_64.whl#md5=337b91f502138ba7fd722803138f6dfd
@@ -63,6 +63,10 @@ mv scikit_learn-0.17.1-cp34-cp34m-manylinux1_x86_64.whl \
 pip3 install scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
 rm scikit_learn-0.17.1-cp34-cp34m-linux_x86_64.whl
 
+# pandas required by tf.learn/inflow
+pip install pandas==0.18.1
+pip3 install pandas==0.18.1
+
 # Benchmark tests require the following:
 pip install psutil
 pip3 install psutil
@@ -72,12 +76,3 @@ pip3 install py-cpuinfo
 # pylint tests require the following:
 pip install pylint
 pip3 install pylint
-
-# Remove packages in /usr/lib/python* that may interfere with packages in
-# /usr/local/lib. These packages may get installed inadvertantly with packages
-# such as apt-get python-pandas. Their older versions can mask the more recent
-# versions installed above with pip and cause test failures.
-rm -rf /usr/lib/python2.7/dist-packages/numpy \
-       /usr/lib/python2.7/dist-packages/scipy \
-       /usr/lib/python3/dist-packages/numpy \
-       /usr/lib/python3/dist-packages/scipy

tensorflow/tools/ci_build/install/install_proto3.sh

@@ -16,19 +16,18 @@
 
 set -e
 
-# Select bazel version.
+# Install protobuf3.
+
+# Select protobuf version.
 PROTOBUF_VERSION="3.0.0-beta-2"
 
-# Install protobuf3.
-mkdir /protobuf
-cd /protobuf
-curl -fSsL -O https://github.com/google/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-cpp-$PROTOBUF_VERSION.tar.gz
-tar zxf protobuf-cpp-$PROTOBUF_VERSION.tar.gz
-cd protobuf-$PROTOBUF_VERSION
-./autogen.sh
-./configure
-make
-make install
-make clean
-ldconfig
-cd /; rm -rf /protobuf
+PROTOBUF_URL="https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip"
+PROTOBUF_ZIP=$(basename "${PROTOBUF_URL}")
+UNZIP_DEST="google-protobuf"
+
+wget -q "${PROTOBUF_URL}"
+unzip "${PROTOBUF_ZIP}" -d "${UNZIP_DEST}"
+cp "${UNZIP_DEST}/protoc" /usr/local/bin/
+
+rm -f "${PROTOBUF_ZIP}"
+rm -rf "${UNZIP_DEST}"

tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh

@@ -34,8 +34,6 @@ make install
 rm -f /usr/bin/swig
 ln -s /usr/local/bin/swig /usr/bin/swig
 
-which swig
-
 popd
 
 rm -rf swig-3.0.8
@@ -61,3 +59,4 @@ pip3.5 install --upgrade scikit-learn
 # Install recent-enough version of wheel for Python 3.5 wheel builds
 pip3.5 install wheel==0.29.0
 
+pip3.5 install --upgrade pandas==0.18.1