update docker containers used for remote CPU/GPU builds and the corresponding script.
PiperOrigin-RevId: 188324090
This commit is contained in:
A. Unique TensorFlower
TensorFlower Gardener
parent
51fd9d70b8
commit
6d44c84bb2
14
tensorflow/tools/ci_build/Dockerfile.rbe.cpu
Normal file
14
tensorflow/tools/ci_build/Dockerfile.rbe.cpu
Normal file
@ -0,0 +1,14 @@
|
||||
FROM launcher.gcr.io/google/rbe-debian8:r322167
|
||||
LABEL maintainer="Yu Yi <yiyu@google.com>"
|
||||
|
||||
# Copy install scripts
|
||||
COPY install/*.sh /install/
|
||||
|
||||
# Setup envvars
|
||||
ENV CC /usr/local/bin/clang
|
||||
ENV CXX /usr/local/bin/clang++
|
||||
ENV AR /usr/bin/ar
|
||||
|
||||
# Run pip install script for RBE Debian8 container.
|
||||
RUN /install/install_pip_packages_remote.sh
|
||||
RUN /install/install_pip_packages.sh
|
||||
@ -16,25 +16,19 @@
|
||||
# Build TensorFlow Docker images for remote build
|
||||
#
|
||||
# Usage:
|
||||
# remote_docker_build.sh -c # docker image for cpu build
|
||||
# remote_docker_build.sh -g # docker image for gpu build
|
||||
|
||||
# ci_rbe_docker_build.sh -c # docker image for cpu build
|
||||
# ci_rbe_docker_build.sh -g # docker image for gpu build
|
||||
|
||||
function main {
|
||||
publish=true
|
||||
cpu_build=false
|
||||
gpu_build=false
|
||||
publish=true
|
||||
publish=false
|
||||
|
||||
script_dir=$(dirname "$(readlink -f "$0")")
|
||||
cd $script_dir
|
||||
|
||||
trap cleanup_on_finish EXIT
|
||||
|
||||
set_script_flags $@
|
||||
|
||||
build_base_image
|
||||
|
||||
build_tf_image
|
||||
|
||||
if [ "$publish" = true ] ; then
|
||||
@ -50,17 +44,14 @@ function set_script_flags {
|
||||
c)
|
||||
cpu_build=true
|
||||
;;
|
||||
f)
|
||||
base_image_build_script=$OPTARG
|
||||
;;
|
||||
g)
|
||||
gpu_build=true
|
||||
;;
|
||||
h)
|
||||
print_usage
|
||||
;;
|
||||
n)
|
||||
publish=false
|
||||
p)
|
||||
publish=true
|
||||
;;
|
||||
*)
|
||||
print_usage "ERROR: unknown option"
|
||||
@ -76,7 +67,6 @@ function print_usage {
|
||||
echo "Usage: $(basename $0) -c | -g [options]"
|
||||
echo " -c build image for CPU build (base image debian8-clang)"
|
||||
echo " -g build image for GPU build (base image nvidia-clang)"
|
||||
echo " -f the script which build the {debian8,nvidia}-clang base image"
|
||||
echo "[option] is one of"
|
||||
echo " -n not publish the locally-built image to GCR;"
|
||||
echo " the build process will publish image to GCR by default"
|
||||
@ -87,54 +77,22 @@ function print_usage {
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
# Build nvidia-cuba-clang base image for GPU image.
|
||||
# For CPU the `clang-debian8` from Cloud Launcher will be used directly:
|
||||
# https://console.cloud.google.com/launcher/details/google/clang-debian8?filter=category:developer-tools&q=clang
|
||||
function build_base_image {
|
||||
if [ "$gpu_build" = true ] ; then
|
||||
base_image="nvidia-cuda"
|
||||
# Run a 2-stage build for clang base image, see
|
||||
# https://github.com/llvm-mirror/llvm/blob/master/docs/Docker.rst
|
||||
$base_image_build_script \
|
||||
--source $base_image \
|
||||
--branch branches/google/stable \
|
||||
--docker-repository ${base_image}-clang --docker-tag "latest" \
|
||||
-p clang -i stage2-install-clang -i stage2-install-clang-headers \
|
||||
-- \
|
||||
-DLLVM_TARGETS_TO_BUILD=Native -DCMAKE_BUILD_TYPE=Release \
|
||||
-DBOOTSTRAP_CMAKE_BUILD_TYPE=Release \
|
||||
-DCLANG_ENABLE_BOOTSTRAP=ON \
|
||||
-DCLANG_BOOTSTRAP_TARGETS="install-clang;install-clang-headers"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
function build_tf_image {
|
||||
if [ "$cpu_build" = true ] ; then
|
||||
dockerfile="Dockerfile.cpu"
|
||||
tf_image="tensorflow-remote"
|
||||
dockerfile="Dockerfile.rbe.cpu"
|
||||
tf_image="tensorflow-rbe-cpu"
|
||||
else
|
||||
dockerfile="Dockerfile.gpu"
|
||||
tf_image="tensorflow-remote-gpu"
|
||||
dockerfile="Dockerfile.rbe.gpu"
|
||||
tf_image="tensorflow-rbe-gpu"
|
||||
fi
|
||||
|
||||
docker build -f $dockerfile -t $tf_image .
|
||||
}
|
||||
|
||||
|
||||
function publish_tf_image {
|
||||
gcr_tf_image="gcr.io/tensorflow/${tf_image}"
|
||||
docker tag $tf_image $gcr_tf_image
|
||||
gcloud docker -- push $gcr_tf_image
|
||||
}
|
||||
|
||||
|
||||
function cleanup_on_finish {
|
||||
cd $script_dir
|
||||
rm -rf $llvm_docker_src
|
||||
docker rmi -f ${base_image}-clang ${base_image}-clang-build
|
||||
}
|
||||
|
||||
|
||||
main $@
|
||||
29
tensorflow/tools/ci_build/install/install_pip_packages_remote.sh
Executable file
29
tensorflow/tools/ci_build/install/install_pip_packages_remote.sh
Executable file
@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
if [ ! -f /usr/bin/x86_64-linux-gnu-gcc ]; then
|
||||
ln -s /usr/local/bin/clang /usr/bin/x86_64-linux-gnu-gcc
|
||||
fi
|
||||
|
||||
pip2 install -U pip
|
||||
pip3 install -U pip
|
||||
pip2 install -U setuptools
|
||||
pip3 install -U setuptools
|
||||
|
||||
# The rest of the pip packages will be installed in
|
||||
# `install_pip_packages.sh`
|
||||
@ -1,27 +0,0 @@
|
||||
FROM launcher.gcr.io/google/clang-debian8:latest
|
||||
|
||||
RUN apt-get update && apt-get --no-install-recommends install -y \
|
||||
binutils \
|
||||
binutils-gold \
|
||||
curl \
|
||||
libstdc++-4.9-dev \
|
||||
python \
|
||||
python-dev \
|
||||
python-numpy \
|
||||
python-pip \
|
||||
unzip \
|
||||
zip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
|
||||
python get-pip.py && \
|
||||
rm get-pip.py
|
||||
|
||||
# Set up grpc
|
||||
RUN pip install --upgrade enum34 futures mock numpy six backports.weakref portpicker && \
|
||||
pip install --pre 'protobuf>=3.0.0a3' && \
|
||||
pip install 'grpcio>=1.1.3'
|
||||
|
||||
# TODO: Set up golang which is compatible with clang
|
||||
|
||||
WORKDIR /botexec
|
||||
@ -1,27 +0,0 @@
|
||||
FROM nvidia-cuda-clang:latest
|
||||
|
||||
RUN apt-get update && apt-get --no-install-recommends install -y \
|
||||
binutils \
|
||||
binutils-gold \
|
||||
curl \
|
||||
libstdc++-4.9-dev \
|
||||
python \
|
||||
python-dev \
|
||||
python-numpy \
|
||||
python-pip \
|
||||
unzip \
|
||||
zip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \
|
||||
python get-pip.py && \
|
||||
rm get-pip.py
|
||||
|
||||
# Set up grpc
|
||||
RUN pip install --upgrade \
|
||||
enum34 futures astor gast mock numpy six \
|
||||
backports.weakref termcolor && \
|
||||
pip install --pre 'protobuf>=3.0.0a3' && \
|
||||
pip install 'grpcio>=1.1.3'
|
||||
|
||||
WORKDIR /botexec
|
||||
2
third_party/toolchains/gpus/cuda/BUILD
vendored
2
third_party/toolchains/gpus/cuda/BUILD
vendored
@ -1272,7 +1272,7 @@ genrule(
|
||||
"cuda/lib/libcupti.so.9.0",
|
||||
],
|
||||
cmd = """
|
||||
if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart.so.9.0.176" "$(@D)/cuda/lib/libcudart.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcublas.so.9.0.282" "$(@D)/cuda/lib/libcublas.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcusolver.so.9.0.176" "$(@D)/cuda/lib/libcusolver.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcurand.so.9.0.176" "$(@D)/cuda/lib/libcurand.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcufft.so.9.0.176" "$(@D)/cuda/lib/libcufft.so.9.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.7.0.5" "$(@D)/cuda/lib/libcudnn.so.7" && cp "/usr/local/cuda-9.0/extras/CUPTI/lib64/libcupti.so.9.0.176" "$(@D)/cuda/lib/libcupti.so.9.0"
|
||||
if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart.so.9.0" "$(@D)/cuda/lib/libcudart.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcublas.so.9.0" "$(@D)/cuda/lib/libcublas.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcusolver.so.9.0" "$(@D)/cuda/lib/libcusolver.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcurand.so.9.0" "$(@D)/cuda/lib/libcurand.so.9.0" && cp "/usr/local/cuda-9.0/targets/x86_64-linux/lib/libcufft.so.9.0" "$(@D)/cuda/lib/libcufft.so.9.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.7" "$(@D)/cuda/lib/libcudnn.so.7" && cp "/usr/local/cuda-9.0/extras/CUPTI/lib64/libcupti.so.9.0" "$(@D)/cuda/lib/libcupti.so.9.0"
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user