Pull out abstractions for devtoolset creation, allowing to put both devtoolset

7 and 8 cross-compilers into the same image.
Update the image creation to include python 3.6, which is our main target for
testing going forward.
Add clang-8 to the image in order to have an option for cuda-clang based builds
based on a single platform.

PiperOrigin-RevId: 252011038
This commit is contained in:
A. Unique TensorFlower 2019-06-07 01:50:30 -07:00 committed by TensorFlower Gardener
parent c8bcbeff7c
commit 3cc8cb8436
4 changed files with 224 additions and 156 deletions

View File

@ -1,156 +0,0 @@
# Dockerfile to build a manylinux 2010 compliant cross-compiler.
#
# Builds a devtoolset-7 gcc/libstdc++ that targets manylinux 2010 compatible
# glibc (2.12) and system libstdc++ (4.4).
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 as devtoolset
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \
cpio \
file \
flex \
g++ \
make \
rpm2cpio \
unar \
wget \
&& \
rm -rf /var/lib/apt/lists/*
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7.
WORKDIR /dt7
# Download binary glibc 2.12 release.
RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.12.1-0ubuntu6_amd64.deb && \
unar libc6_2.12.1-0ubuntu6_amd64.deb && \
tar xvzf libc6_2.12.1-0ubuntu6_amd64/data.tar.gz && \
rm -rf libc6_2.12.1-0ubuntu6_amd64.deb libc6_2.12.1-0ubuntu6_amd64
RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.12.1-0ubuntu6_amd64.deb && \
unar libc6-dev_2.12.1-0ubuntu6_amd64.deb && \
tar xvzf libc6-dev_2.12.1-0ubuntu6_amd64/data.tar.gz && \
rm -rf libc6-dev_2.12.1-0ubuntu6_amd64.deb libc6-dev_2.12.1-0ubuntu6_amd64
# Put the current kernel headers from ubuntu in place.
RUN ln -s /usr/include/linux /dt7/usr/include/linux
RUN ln -s /usr/include/asm-generic /dt7/usr/include/asm-generic
RUN ln -s /usr/include/x86_64-linux-gnu/asm /dt7/usr/include/asm
# Symlinks in the binary distribution are set up for installation in /usr, we
# need to fix up all the links to stay within /dt7.
ADD devtoolset/fixlinks.sh fixlinks.sh
RUN ./fixlinks.sh /dt7
# Patch to allow non-glibc 2.12 compatible builds to work.
RUN sed -i '54i#define TCP_USER_TIMEOUT 18' /dt7/usr/include/netinet/tcp.h
# Download binary libstdc++ 4.4 release.
WORKDIR /libstdc++-4.4
RUN wget http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.4/libstdc++6_4.4.3-4ubuntu5_amd64.deb && \
unar libstdc++6_4.4.3-4ubuntu5_amd64.deb && \
tar xvzf libstdc++6_4.4.3-4ubuntu5_amd64/data.tar.gz && \
rm -rf libstdc++6_4.4.3-4ubuntu5_amd64.deb libstdc++6_4.4.3-4ubuntu5_amd64
# We only need the shared library, as we're going to develop against the
# libstdc++ provided by devtoolset-7.
RUN cp ./usr/lib/libstdc++.so.6.0.13 /dt7/usr/lib
# Build a devtoolset-7 cross-compiler based on our glibc 2.12 sysroot setup.
WORKDIR /dts/src
RUN wget http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-7/devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm
RUN rpm2cpio devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm |cpio -idmv
RUN tar xvjf gcc-7.3.1-20180303.tar.bz2 --strip 1
# Apply the devtoolset patches to gcc.
ADD devtoolset/rpm-patch.sh rpm-patch.sh
RUN ./rpm-patch.sh gcc.spec
RUN ./contrib/download_prerequisites
WORKDIR /dts/build
RUN /dts/src/configure \
--prefix=/dt7/usr \
--with-sysroot=/dt7 \
--disable-bootstrap \
--disable-libmpx \
--disable-libsanitizer \
--disable-libunwind-exceptions \
--disable-libunwind-exceptions \
--disable-lto \
--disable-multilib \
--enable-__cxa_atexit \
--enable-gnu-indirect-function \
--enable-gnu-unique-object \
--enable-initfini-array \
--enable-languages="c,c++" \
--enable-linker-build-id \
--enable-plugin \
--enable-shared \
--enable-threads=posix \
--with-default-libstdcxx-abi=gcc4-compatible \
--with-gcc-major-version-only \
--with-linker-hash-style=gnu \
--with-tune=generic \
&& \
make -j 42 && \
make install
# Create the devtoolset libstdc++ linkerscript that links dynamically against
# the system libstdc++ 4.4 and provides all other symbols statically.
RUN mv /dt7/usr/lib/libstdc++.so.6.0.24 /dt7/usr/lib/libstdc++.so.6.0.24.backup
RUN echo "OUTPUT_FORMAT(elf64-x86-64)\nINPUT ( libstdc++.so.6.0.13 -lstdc++_nonshared44 )" \
> /dt7/usr/lib/libstdc++.so.6.0.24
RUN cp ./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared44.a \
/dt7/usr/lib
# Link in architecture specific includes from the system; note that we cannot
# link in the whole x86_64-linux-gnu folder, as otherwise we're overlaying
# system gcc paths that we do not want to find.
# TODO(klimek): Automate linking in all non-gcc / non-kernel include
# directories.
RUN mkdir -p /dt7/usr/include/x86_64-linux-gnu
RUN ln -s /usr/include/x86_64-linux-gnu/python3.5m /dt7/usr/include/x86_64-linux-gnu/python3.5m
# TODO(klimek): Split up into two different docker images.
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
COPY --from=devtoolset /dt7 /dt7
# Copy and run the install scripts.
COPY install/*.sh /install/
ARG DEBIAN_FRONTEND=noninteractive
RUN /install/install_bootstrap_deb_packages.sh
RUN add-apt-repository -y ppa:openjdk-r/ppa && \
add-apt-repository -y ppa:george-edison55/cmake-3.x
RUN /install/install_deb_packages.sh
RUN /install/install_pip_packages.sh
RUN /install/install_golang.sh
RUN /install/install_bazel.sh
# Install TensorRT.
RUN apt-get update && apt-get install -y \
libnvinfer-dev=5.1.5-1+cuda10.0 \
libnvinfer5=5.1.5-1+cuda10.0 \
&& \
rm -rf /var/lib/apt/lists/*
# Build the pip package.
WORKDIR /tf
RUN git clone https://github.com/tensorflow/tensorflow .
ENV TF_NEED_GCP=1 \
TF_NEED_HDFS=1 \
TF_NEED_CUDA=1 \
GCC_HOST_COMPILER_PATH=/dt7/usr/bin/gcc \
PATH=/dt7/usr/bin:$PATH \
PYTHON_BIN_PATH=/usr/bin/python3.5 \
TF_NEED_TENSORRT=1
ADD devtoolset/platlib.patch platlib.patch
RUN patch -p1 < platlib.patch
RUN yes "" | ./configure
RUN bazel build --config=opt tensorflow/tools/pip_package:build_pip_package --nodistinct_host_configuration
RUN ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tf-wheel

View File

@ -0,0 +1,68 @@
# Dockerfile to build a manylinux 2010 compliant cross-compiler.
#
# Builds a devtoolset gcc/libstdc++ that targets manylinux 2010 compatible
# glibc (2.12) and system libstdc++ (4.4).
#
# To push a new version, run:
# $ docker build -f Dockerfile.rbe.cuda10.0-cudnn7-ubuntu16.04-manylinux2010 \
# --tag "gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu16.04-manylinux2010" .
# $ docker push gcr.io/tensorflow-testing/nosla-cuda10.0-cudnn7-ubuntu16.04-manylinux2010
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04 as devtoolset
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \
cpio \
file \
flex \
g++ \
make \
rpm2cpio \
unar \
wget \
&& \
rm -rf /var/lib/apt/lists/*
ADD devtoolset/fixlinks.sh fixlinks.sh
ADD devtoolset/build_devtoolset.sh build_devtoolset.sh
ADD devtoolset/rpm-patch.sh rpm-patch.sh
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7.
RUN /build_devtoolset.sh devtoolset-7 /dt7
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-8 in /dt8.
RUN /build_devtoolset.sh devtoolset-8 /dt8
# TODO(klimek): Split up into two different docker images.
FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
COPY --from=devtoolset /dt7 /dt7
COPY --from=devtoolset /dt8 /dt8
# Install TensorRT.
RUN apt-get update && apt-get install -y \
libnvinfer-dev=5.1.5-1+cuda10.0 \
libnvinfer5=5.1.5-1+cuda10.0 \
&& \
rm -rf /var/lib/apt/lists/*
# Copy and run the install scripts.
COPY install/*.sh /install/
ARG DEBIAN_FRONTEND=noninteractive
RUN /install/install_bootstrap_deb_packages.sh
RUN /install/install_deb_packages.sh
RUN /install/install_clang.sh
RUN /install/install_bazel.sh
# Install python 3.6.
RUN add-apt-repository ppa:jonathonf/python-3.6 && \
apt-get update && apt-get install -y \
python3.6 python3.6-dev python3-pip python3.6-venv && \
rm -rf /var/lib/apt/lists/* && \
python3.6 -m pip install pip --upgrade && \
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 0
RUN /install/install_pip_packages.sh
# TODO(klimek): Figure out a better way to get the right include paths
# forwarded when we install new packages.
RUN ln -s "/usr/include/x86_64-linux-gnu/python3.6m" "/dt7/usr/include/x86_64-linux-gnu/python3.6m"
RUN ln -s "/usr/include/x86_64-linux-gnu/python3.6m" "/dt8/usr/include/x86_64-linux-gnu/python3.6m"

View File

@ -0,0 +1,135 @@
#!/bin/bash -eu
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
#
# Builds a devtoolset cross-compiler targeting manylinux 2010 (glibc 2.12 /
# libstdc++ 4.4).
VERSION="$1"
TARGET="$2"
case "${VERSION}" in
devtoolset-7)
LIBSTDCXX_VERSION="6.0.24"
;;
devtoolset-8)
LIBSTDCXX_VERSION="6.0.25"
;;
*)
echo "Usage: $0 {devtoolset-7|devtoolset-8} <target-directory>"
exit 1
;;
esac
mkdir -p "${TARGET}"
# Download binary glibc 2.12 release.
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6_2.12.1-0ubuntu6_amd64.deb" && \
unar "libc6_2.12.1-0ubuntu6_amd64.deb" && \
tar -C "${TARGET}" -xvzf "libc6_2.12.1-0ubuntu6_amd64/data.tar.gz" && \
rm -rf "libc6_2.12.1-0ubuntu6_amd64.deb" "libc6_2.12.1-0ubuntu6_amd64"
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \
unar "libc6-dev_2.12.1-0ubuntu6_amd64.deb" && \
tar -C "${TARGET}" -xvzf "libc6-dev_2.12.1-0ubuntu6_amd64/data.tar.gz" && \
rm -rf "libc6-dev_2.12.1-0ubuntu6_amd64.deb" "libc6-dev_2.12.1-0ubuntu6_amd64"
# Put the current kernel headers from ubuntu in place.
ln -s "/usr/include/linux" "/${TARGET}/usr/include/linux"
ln -s "/usr/include/asm-generic" "/${TARGET}/usr/include/asm-generic"
ln -s "/usr/include/x86_64-linux-gnu/asm" "/${TARGET}/usr/include/asm"
# Symlinks in the binary distribution are set up for installation in /usr, we
# need to fix up all the links to stay within /${TARGET}.
/fixlinks.sh "/${TARGET}"
# Patch to allow non-glibc 2.12 compatible builds to work.
sed -i '54i#define TCP_USER_TIMEOUT 18' "/${TARGET}/usr/include/netinet/tcp.h"
# Download binary libstdc++ 4.4 release we are going to link against.
# We only need the shared library, as we're going to develop against the
# libstdc++ provided by devtoolset.
wget "http://old-releases.ubuntu.com/ubuntu/pool/main/g/gcc-4.4/libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \
unar "libstdc++6_4.4.3-4ubuntu5_amd64.deb" && \
tar -C "/${TARGET}" -xvzf "libstdc++6_4.4.3-4ubuntu5_amd64/data.tar.gz" "./usr/lib/libstdc++.so.6.0.13" && \
rm -rf "libstdc++6_4.4.3-4ubuntu5_amd64.deb" "libstdc++6_4.4.3-4ubuntu5_amd64"
mkdir -p "${TARGET}-src"
cd "${TARGET}-src"
# Build a devtoolset cross-compiler based on our glibc 2.12 sysroot setup.
case "${VERSION}" in
devtoolset-7)
wget "http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-7/devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm"
rpm2cpio "devtoolset-7-gcc-7.3.1-5.15.el6.src.rpm" |cpio -idmv
tar -xvjf "gcc-7.3.1-20180303.tar.bz2" --strip 1
;;
devtoolset-8)
wget "http://vault.centos.org/centos/6/sclo/Source/rh/devtoolset-8/devtoolset-8-gcc-8.2.1-3.el6.src.rpm"
rpm2cpio "devtoolset-8-gcc-8.2.1-3.el6.src.rpm" |cpio -idmv
tar -xvf "gcc-8.2.1-20180905.tar.xz" --strip 1
;;
esac
# Apply the devtoolset patches to gcc.
/rpm-patch.sh "gcc.spec"
./contrib/download_prerequisites
mkdir -p "${TARGET}-build"
cd "${TARGET}-build"
"${TARGET}-src/configure" \
--prefix=/"${TARGET}/usr" \
--with-sysroot="/${TARGET}" \
--disable-bootstrap \
--disable-libmpx \
--disable-libsanitizer \
--disable-libunwind-exceptions \
--disable-libunwind-exceptions \
--disable-lto \
--disable-multilib \
--enable-__cxa_atexit \
--enable-gnu-indirect-function \
--enable-gnu-unique-object \
--enable-initfini-array \
--enable-languages="c,c++" \
--enable-linker-build-id \
--enable-plugin \
--enable-shared \
--enable-threads=posix \
--with-default-libstdcxx-abi="gcc4-compatible" \
--with-gcc-major-version-only \
--with-linker-hash-style="gnu" \
--with-tune="generic" \
&& \
make -j 42 && \
make install
# Create the devtoolset libstdc++ linkerscript that links dynamically against
# the system libstdc++ 4.4 and provides all other symbols statically.
mv "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}" \
"/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}.backup"
echo -e "OUTPUT_FORMAT(elf64-x86-64)\nINPUT ( libstdc++.so.6.0.13 -lstdc++_nonshared44 )" \
> "/${TARGET}/usr/lib/libstdc++.so.${LIBSTDCXX_VERSION}"
cp "./x86_64-pc-linux-gnu/libstdc++-v3/src/.libs/libstdc++_nonshared44.a" \
"/${TARGET}/usr/lib"
# Link in architecture specific includes from the system; note that we cannot
# link in the whole x86_64-linux-gnu folder, as otherwise we're overlaying
# system gcc paths that we do not want to find.
# TODO(klimek): Automate linking in all non-gcc / non-kernel include
# directories.
mkdir -p "/${TARGET}/usr/include/x86_64-linux-gnu"
ln -s "/usr/include/x86_64-linux-gnu/python3.5m" "/${TARGET}/usr/include/x86_64-linux-gnu/python3.5m"

View File

@ -0,0 +1,21 @@
#!/bin/bash -eu
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
DIST="$(grep "DISTRIB_CODENAME" /etc/lsb-release |sed 's,.*=,,')"
wget -O - "https://apt.llvm.org/llvm-snapshot.gpg.key"| apt-key add -
add-apt-repository "deb http://apt.llvm.org/${DIST}/ llvm-toolchain-${DIST}-8 main"
apt-get update && apt-get install -y clang-8 && \
rm -rf /var/lib/apt/lists/*