From 3cc8cb8436911b96c57cea3961ff7053985ae837 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Jun 2019 01:50:30 -0700 Subject: [PATCH] Pull out abstractions for devtoolset creation, allowing to put both devtoolset 7 and 8 cross-compilers into the same image. Update the image creation to include python 3.6, which is our main target for testing going forward. Add clang-8 to the image in order to have an option for cuda-clang based builds based on a single platform. PiperOrigin-RevId: 252011038 --- ...dnn7-ubuntu16.04-devtoolset7-manylinux2010 | 156 ------------------ ....cuda10.0-cudnn7-ubuntu16.04-manylinux2010 | 68 ++++++++ .../ci_build/devtoolset/build_devtoolset.sh | 135 +++++++++++++++ .../tools/ci_build/install/install_clang.sh | 21 +++ 4 files changed, 224 insertions(+), 156 deletions(-) delete mode 100644 tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-devtoolset7-manylinux2010 create mode 100644 tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-manylinux2010 create mode 100755 tensorflow/tools/ci_build/devtoolset/build_devtoolset.sh create mode 100755 tensorflow/tools/ci_build/install/install_clang.sh diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-devtoolset7-manylinux2010 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-devtoolset7-manylinux2010 deleted file mode 100644 index 84603c79795..00000000000 --- a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-devtoolset7-manylinux2010 +++ /dev/null @@ -1,156 +0,0 @@ -# Dockerfile to build a manylinux 2010 compliant cross-compiler. -# -# Builds a devtoolset-7 gcc/libstdc++ that targets manylinux 2010 compatible -# glibc (2.12) and system libstdc++ (4.4). - -FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 as devtoolset - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y \ - cpio \ - file \ - flex \ - g++ \ - make \ - rpm2cpio \ - unar \ - wget \ - && \ - rm -rf /var/lib/apt/lists/* - - -# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7. -WORKDIR /dt7 - -# Download binary glibc 2.12 release. -RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.12.1-0ubuntu6_amd64.deb && \ - unar libc6_2.12.1-0ubuntu6_amd64.deb && \ - tar xvzf libc6_2.12.1-0ubuntu6_amd64/data.tar.gz && \ - rm -rf libc6_2.12.1-0ubuntu6_amd64.deb libc6_2.12.1-0ubuntu6_amd64 -RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.12.1-0ubuntu6_amd64.deb && \ - unar libc6-dev_2.12.1-0ubuntu6_amd64.deb && \ - tar xvzf libc6-dev_2.12.1-0ubuntu6_amd64/data.tar.gz && \ - rm -rf libc6-dev_2.12.1-0ubuntu6_amd64.deb libc6-dev_2.12.1-0ubuntu6_amd64 - -# Put the current kernel headers from ubuntu in place. -RUN ln -s /usr/include/linux /dt7/usr/include/linux -RUN ln -s /usr/include/asm-generic /dt7/usr/include/asm-generic -RUN ln -s /usr/include/x86_64-linux-gnu/asm /dt7/usr/include/asm - -# Symlinks in the binary distribution are set up for installation in /usr, we -# need to fix up all the links to stay within /dt7. -ADD devtoolset/fixlinks.sh fixlinks.sh -RUN ./fixlinks.sh /dt7 - -# Patch to allow non-glibc 2.12 compatible builds to work. -RUN sed -i '54i#define TCP_USER_TIMEOUT 18' /dt7/usr/include/netinet/tcp.h - -# Download binary libstdc++ 4.4 release. -WORKDIR /libstdc++-4.4 -RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.4/libstdc++6_4.4.3-4ubuntu5_amd64.deb && \ - unar libstdc++6_4.4.3-4ubuntu5_amd64.deb && \ - tar xvzf libstdc++6_4.4.3-4ubuntu5_amd64/data.tar.gz && \ - rm -rf libstdc++6_4.4.3-4ubuntu5_amd64.deb libstdc++6_4.4.3-4ubuntu5_amd64 - -# We only need the shared library, as we're going to develop against the -# libstdc++ provided by devtoolset-7. -RUN cp ./usr/lib/libstdc++.so.6.0.13 /dt7/usr/lib - -# Build a devtoolset-7 cross-compiler based on our glibc 2.12 sysroot setup. -WORKDIR /dts/src -RUN wget http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-7/devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm -RUN rpm2cpio devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm |cpio -idmv -RUN tar xvjf gcc-7.3.1-20180303.tar.bz2 --strip 1 - -# Apply the devtoolset patches to gcc. -ADD devtoolset/rpm-patch.sh rpm-patch.sh -RUN ./rpm-patch.sh gcc.spec - -RUN ./contrib/download_prerequisites - -WORKDIR /dts/build -RUN /dts/src/configure \ - --prefix=/dt7/usr \ - --with-sysroot=/dt7 \ - --disable-bootstrap \ - --disable-libmpx \ - --disable-libsanitizer \ - --disable-libunwind-exceptions \ - --disable-libunwind-exceptions \ - --disable-lto \ - --disable-multilib \ - --enable-__cxa_atexit \ - --enable-gnu-indirect-function \ - --enable-gnu-unique-object \ - --enable-initfini-array \ - --enable-languages="c,c++" \ - --enable-linker-build-id \ - --enable-plugin \ - --enable-shared \ - --enable-threads=posix \ - --with-default-libstdcxx-abi=gcc4-compatible \ - --with-gcc-major-version-only \ - --with-linker-hash-style=gnu \ - --with-tune=generic \ - && \ - make -j 42 && \ - make install - -# Create the devtoolset libstdc++ linkerscript that links dynamically against -# the system libstdc++ 4.4 and provides all other symbols statically. -RUN mv /dt7/usr/lib/libstdc++.so.6.0.24 /dt7/usr/lib/libstdc++.so.6.0.24.backup -RUN echo "OUTPUT_FORMAT(elf64-x86-64)\nINPUT ( libstdc++.so.6.0.13 -lstdc++_nonshared44 )" \ - > /dt7/usr/lib/libstdc++.so.6.0.24 -RUN cp ./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared44.a \ - /dt7/usr/lib - -# Link in architecture specific includes from the system; note that we cannot -# link in the whole x86_64-linux-gnu folder, as otherwise we're overlaying -# system gcc paths that we do not want to find. -# TODO(klimek): Automate linking in all non-gcc / non-kernel include -# directories. -RUN mkdir -p /dt7/usr/include/x86_64-linux-gnu -RUN ln -s /usr/include/x86_64-linux-gnu/python3.5m /dt7/usr/include/x86_64-linux-gnu/python3.5m - -# TODO(klimek): Split up into two different docker images. -FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 -COPY --from=devtoolset /dt7 /dt7 - -# Copy and run the install scripts. -COPY install/*.sh /install/ -ARG DEBIAN_FRONTEND=noninteractive -RUN /install/install_bootstrap_deb_packages.sh -RUN add-apt-repository -y ppa:openjdk-r/ppa && \ - add-apt-repository -y ppa:george-edison55/cmake-3.x -RUN /install/install_deb_packages.sh -RUN /install/install_pip_packages.sh -RUN /install/install_golang.sh -RUN /install/install_bazel.sh - -# Install TensorRT. -RUN apt-get update && apt-get install -y \ - libnvinfer-dev=5.1.5-1+cuda10.0 \ - libnvinfer5=5.1.5-1+cuda10.0 \ - && \ - rm -rf /var/lib/apt/lists/* - -# Build the pip package. -WORKDIR /tf -RUN git clone https://github.com/tensorflow/tensorflow . - -ENV TF_NEED_GCP=1 \ - TF_NEED_HDFS=1 \ - TF_NEED_CUDA=1 \ - GCC_HOST_COMPILER_PATH=/dt7/usr/bin/gcc \ - PATH=/dt7/usr/bin:$PATH \ - PYTHON_BIN_PATH=/usr/bin/python3.5 \ - TF_NEED_TENSORRT=1 - -ADD devtoolset/platlib.patch platlib.patch -RUN patch -p1 < platlib.patch - -RUN yes "" | ./configure - -RUN bazel build --config=opt tensorflow/tools/pip_package:build_pip_package --nodistinct_host_configuration - -RUN ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tf-wheel diff --git a/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-manylinux2010 b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-manylinux2010 new file mode 100644 index 00000000000..7870839dae3 --- /dev/null +++ b/tensorflow/tools/ci_build/Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-manylinux2010 @@ -0,0 +1,68 @@ +# Dockerfile to build a manylinux 2010 compliant cross-compiler. +# +# Builds a devtoolset gcc/libstdc++ that targets manylinux 2010 compatible +# glibc (2.12) and system libstdc++ (4.4). +# +# To push a new version, run: +# $ docker build -f Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-manylinux2010 \ +# --tag "gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu16.04-manylinux2010" . +# $ docker push gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu16.04-manylinux2010 + +FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 as devtoolset + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y \ + cpio \ + file \ + flex \ + g++ \ + make \ + rpm2cpio \ + unar \ + wget \ + && \ + rm -rf /var/lib/apt/lists/* + +ADD devtoolset/fixlinks.sh fixlinks.sh +ADD devtoolset/build_devtoolset.sh build_devtoolset.sh +ADD devtoolset/rpm-patch.sh rpm-patch.sh + +# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7. +RUN /build_devtoolset.sh devtoolset-7 /dt7 +# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-8 in /dt8. +RUN /build_devtoolset.sh devtoolset-8 /dt8 + +# TODO(klimek): Split up into two different docker images. +FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 +COPY --from=devtoolset /dt7 /dt7 +COPY --from=devtoolset /dt8 /dt8 + +# Install TensorRT. +RUN apt-get update && apt-get install -y \ + libnvinfer-dev=5.1.5-1+cuda10.0 \ + libnvinfer5=5.1.5-1+cuda10.0 \ + && \ + rm -rf /var/lib/apt/lists/* + +# Copy and run the install scripts. +COPY install/*.sh /install/ +ARG DEBIAN_FRONTEND=noninteractive +RUN /install/install_bootstrap_deb_packages.sh +RUN /install/install_deb_packages.sh +RUN /install/install_clang.sh +RUN /install/install_bazel.sh + +# Install python 3.6. +RUN add-apt-repository ppa:jonathonf/python-3.6 && \ + apt-get update && apt-get install -y \ + python3.6 python3.6-dev python3-pip python3.6-venv && \ + rm -rf /var/lib/apt/lists/* && \ + python3.6 -m pip install pip --upgrade && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 0 + +RUN /install/install_pip_packages.sh + +# TODO(klimek): Figure out a better way to get the right include paths +# forwarded when we install new packages. +RUN ln -s "/usr/include/x86_64-linux-gnu/python3.6m" "/dt7/usr/include/x86_64-linux-gnu/python3.6m" +RUN ln -s "/usr/include/x86_64-linux-gnu/python3.6m" "/dt8/usr/include/x86_64-linux-gnu/python3.6m" diff --git a/tensorflow/tools/ci_build/devtoolset/build_devtoolset.sh b/tensorflow/tools/ci_build/devtoolset/build_devtoolset.sh new file mode 100755 index 00000000000..4343438f205 --- /dev/null +++ b/tensorflow/tools/ci_build/devtoolset/build_devtoolset.sh @@ -0,0 +1,135 @@ +#!/bin/bash -eu +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# Builds a devtoolset cross-compiler targeting manylinux 2010 (glibc 2.12 / +# libstdc++ 4.4). + +VERSION="$1" +TARGET="$2" + +case "${VERSION}" in +devtoolset-7) + LIBSTDCXX_VERSION="6.0.24" + ;; +devtoolset-8) + LIBSTDCXX_VERSION="6.0.25" + ;; +*) + echo "Usage: $0 {devtoolset-7|devtoolset-8} " + exit 1 + ;; +esac + +mkdir -p "${TARGET}" +# Download binary glibc 2.12 release. +wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.12.1-0ubuntu6_amd64.deb" && \ + unar "libc6_2.12.1-0ubuntu6_amd64.deb" && \ + tar -C "${TARGET}" -xvzf "libc6_2.12.1-0ubuntu6_amd64/data.tar.gz" && \ + rm -rf "libc6_2.12.1-0ubuntu6_amd64.deb" "libc6_2.12.1-0ubuntu6_amd64" +wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \ + unar "libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \ + tar -C "${TARGET}" -xvzf "libc6-dev_2.12.1-0ubuntu6_amd64/data.tar.gz" && \ + rm -rf "libc6-dev_2.12.1-0ubuntu6_amd64.deb" "libc6-dev_2.12.1-0ubuntu6_amd64" + +# Put the current kernel headers from ubuntu in place. +ln -s "/usr/include/linux" "/${TARGET}/usr/include/linux" +ln -s "/usr/include/asm-generic" "/${TARGET}/usr/include/asm-generic" +ln -s "/usr/include/x86_64-linux-gnu/asm" "/${TARGET}/usr/include/asm" + +# Symlinks in the binary distribution are set up for installation in /usr, we +# need to fix up all the links to stay within /${TARGET}. +/fixlinks.sh "/${TARGET}" + +# Patch to allow non-glibc 2.12 compatible builds to work. +sed -i '54i#define TCP_USER_TIMEOUT 18' "/${TARGET}/usr/include/netinet/tcp.h" + +# Download binary libstdc++ 4.4 release we are going to link against. +# We only need the shared library, as we're going to develop against the +# libstdc++ provided by devtoolset. +wget "http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.4/libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \ + unar "libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \ + tar -C "/${TARGET}" -xvzf "libstdc++6_4.4.3-4ubuntu5_amd64/data.tar.gz" "./usr/lib/libstdc++.so.6.0.13" && \ + rm -rf "libstdc++6_4.4.3-4ubuntu5_amd64.deb" "libstdc++6_4.4.3-4ubuntu5_amd64" + +mkdir -p "${TARGET}-src" +cd "${TARGET}-src" + +# Build a devtoolset cross-compiler based on our glibc 2.12 sysroot setup. + +case "${VERSION}" in +devtoolset-7) + wget "http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-7/devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm" + rpm2cpio "devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm" |cpio -idmv + tar -xvjf "gcc-7.3.1-20180303.tar.bz2" --strip 1 + ;; +devtoolset-8) + wget "http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-8/devtoolset-8-gcc-8.2.1-3.el6.src.rpm" + rpm2cpio "devtoolset-8-gcc-8.2.1-3.el6.src.rpm" |cpio -idmv + tar -xvf "gcc-8.2.1-20180905.tar.xz" --strip 1 + ;; +esac + +# Apply the devtoolset patches to gcc. +/rpm-patch.sh "gcc.spec" + +./contrib/download_prerequisites + +mkdir -p "${TARGET}-build" +cd "${TARGET}-build" + +"${TARGET}-src/configure" \ + --prefix=/"${TARGET}/usr" \ + --with-sysroot="/${TARGET}" \ + --disable-bootstrap \ + --disable-libmpx \ + --disable-libsanitizer \ + --disable-libunwind-exceptions \ + --disable-libunwind-exceptions \ + --disable-lto \ + --disable-multilib \ + --enable-__cxa_atexit \ + --enable-gnu-indirect-function \ + --enable-gnu-unique-object \ + --enable-initfini-array \ + --enable-languages="c,c++" \ + --enable-linker-build-id \ + --enable-plugin \ + --enable-shared \ + --enable-threads=posix \ + --with-default-libstdcxx-abi="gcc4-compatible" \ + --with-gcc-major-version-only \ + --with-linker-hash-style="gnu" \ + --with-tune="generic" \ + && \ + make -j 42 && \ + make install + +# Create the devtoolset libstdc++ linkerscript that links dynamically against +# the system libstdc++ 4.4 and provides all other symbols statically. +mv "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}" \ + "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}.backup" +echo -e "OUTPUT_FORMAT(elf64-x86-64)\nINPUT ( libstdc++.so.6.0.13 -lstdc++_nonshared44 )" \ + > "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}" +cp "./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared44.a" \ + "/${TARGET}/usr/lib" + +# Link in architecture specific includes from the system; note that we cannot +# link in the whole x86_64-linux-gnu folder, as otherwise we're overlaying +# system gcc paths that we do not want to find. +# TODO(klimek): Automate linking in all non-gcc / non-kernel include +# directories. +mkdir -p "/${TARGET}/usr/include/x86_64-linux-gnu" +ln -s "/usr/include/x86_64-linux-gnu/python3.5m" "/${TARGET}/usr/include/x86_64-linux-gnu/python3.5m" diff --git a/tensorflow/tools/ci_build/install/install_clang.sh b/tensorflow/tools/ci_build/install/install_clang.sh new file mode 100755 index 00000000000..18eec53b476 --- /dev/null +++ b/tensorflow/tools/ci_build/install/install_clang.sh @@ -0,0 +1,21 @@ +#!/bin/bash -eu +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +DIST="$(grep "DISTRIB_CODENAME" /etc/lsb-release |sed 's,.*=,,')" +wget -O - "https://apt.llvm.org/llvm-snapshot.gpg.key"| apt-key add - +add-apt-repository "deb http://apt.llvm.org/${DIST}/ llvm-toolchain-${DIST}-8 main" +apt-get update && apt-get install -y clang-8 && \ + rm -rf /var/lib/apt/lists/*