Adding OneDNN+MPI+Horovod partials and dockerfiles
This commit is contained in:
Abolfazl Shahbazi
parent
34f2782a79
commit
a7580dc7f2
@ -1,183 +0,0 @@
|
|||||||
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
# ============================================================================
|
|
||||||
#
|
|
||||||
# THIS IS A GENERATED DOCKERFILE.
|
|
||||||
#
|
|
||||||
# This file was assembled from multiple pieces, whose use is documented
|
|
||||||
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
|
||||||
# for more information.
|
|
||||||
|
|
||||||
ARG UBUNTU_VERSION=18.04
|
|
||||||
|
|
||||||
FROM ubuntu:${UBUNTU_VERSION} AS base
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
||||||
build-essential \
|
|
||||||
curl \
|
|
||||||
git \
|
|
||||||
libcurl3-dev \
|
|
||||||
libfreetype6-dev \
|
|
||||||
libhdf5-serial-dev \
|
|
||||||
libzmq3-dev \
|
|
||||||
pkg-config \
|
|
||||||
rsync \
|
|
||||||
software-properties-common \
|
|
||||||
sudo \
|
|
||||||
unzip \
|
|
||||||
zip \
|
|
||||||
zlib1g-dev \
|
|
||||||
openjdk-8-jdk \
|
|
||||||
openjdk-8-jre-headless \
|
|
||||||
&& \
|
|
||||||
apt-get clean && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
ENV CI_BUILD_PYTHON python
|
|
||||||
|
|
||||||
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
|
|
||||||
ARG CACHE_STOP=1
|
|
||||||
# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
|
|
||||||
ARG CHECKOUT_TF_SRC=0
|
|
||||||
# In case of Python 2.7+ we need to add passwd entries for user and group id
|
|
||||||
RUN chmod a+w /etc/passwd /etc/group
|
|
||||||
RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
|
|
||||||
|
|
||||||
# See http://bugs.python.org/issue19846
|
|
||||||
ENV LANG C.UTF-8
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
python3 \
|
|
||||||
python3-pip
|
|
||||||
|
|
||||||
RUN python3 -m pip --no-cache-dir install --upgrade \
|
|
||||||
pip \
|
|
||||||
setuptools
|
|
||||||
|
|
||||||
# Some TF tools expect a "python" binary
|
|
||||||
RUN ln -s $(which python3) /usr/local/bin/python
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
|
||||||
build-essential \
|
|
||||||
curl \
|
|
||||||
git \
|
|
||||||
wget \
|
|
||||||
openjdk-8-jdk \
|
|
||||||
python3-dev \
|
|
||||||
virtualenv \
|
|
||||||
swig
|
|
||||||
|
|
||||||
RUN python3 -m pip --no-cache-dir install \
|
|
||||||
Pillow \
|
|
||||||
h5py \
|
|
||||||
keras_preprocessing \
|
|
||||||
matplotlib \
|
|
||||||
mock \
|
|
||||||
'numpy<1.19.0' \
|
|
||||||
scipy \
|
|
||||||
sklearn \
|
|
||||||
pandas \
|
|
||||||
future \
|
|
||||||
portpicker \
|
|
||||||
enum34
|
|
||||||
|
|
||||||
# Install bazel
|
|
||||||
ARG BAZEL_VERSION=3.1.0
|
|
||||||
RUN mkdir /bazel && \
|
|
||||||
wget -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
|
||||||
wget -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
|
||||||
chmod +x /bazel/installer.sh && \
|
|
||||||
/bazel/installer.sh && \
|
|
||||||
rm -f /bazel/installer.sh
|
|
||||||
|
|
||||||
# install libnuma, openssh, wget
|
|
||||||
RUN ( apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
|
||||||
libnuma-dev \
|
|
||||||
openssh-server \
|
|
||||||
openssh-client \
|
|
||||||
wget && \
|
|
||||||
apt-get clean && \
|
|
||||||
rm -rf /var/lib/apt/lists/* ) || \
|
|
||||||
( yum -y update && yum -y install \
|
|
||||||
numactl-devel \
|
|
||||||
openssh-server \
|
|
||||||
openssh-clients \
|
|
||||||
wget && \
|
|
||||||
yum clean all ) || \
|
|
||||||
( echo "Unsupported Linux distribution. Aborting!" && exit 1 )
|
|
||||||
|
|
||||||
# Install Open MPI
|
|
||||||
# download realese version from official website as openmpi github master is not always stable
|
|
||||||
ARG OPENMPI_VERSION=openmpi-4.0.0
|
|
||||||
ARG OPENMPI_DOWNLOAD_URL=https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.0.tar.gz
|
|
||||||
RUN mkdir /tmp/openmpi && \
|
|
||||||
cd /tmp/openmpi && \
|
|
||||||
wget ${OPENMPI_DOWNLOAD_URL} && \
|
|
||||||
tar zxf ${OPENMPI_VERSION}.tar.gz && \
|
|
||||||
cd ${OPENMPI_VERSION} && \
|
|
||||||
./configure --enable-orterun-prefix-by-default && \
|
|
||||||
make -j $(nproc) all && \
|
|
||||||
make install && \
|
|
||||||
ldconfig && \
|
|
||||||
rm -rf /tmp/openmpi
|
|
||||||
|
|
||||||
# Create a wrapper for OpenMPI to allow running as root by default
|
|
||||||
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
|
|
||||||
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
|
|
||||||
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
|
|
||||||
chmod a+x /usr/local/bin/mpirun
|
|
||||||
|
|
||||||
# Configure OpenMPI to run good defaults:
|
|
||||||
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
|
|
||||||
|
|
||||||
# Install OpenSSH for MPI to communicate between containers
|
|
||||||
RUN mkdir -p /var/run/sshd
|
|
||||||
|
|
||||||
# Allow OpenSSH to talk to containers without asking for confirmation
|
|
||||||
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
|
||||||
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
|
||||||
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
|
||||||
|
|
||||||
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
|
||||||
ARG CHECKOUT_HOROVOD_SRC=0
|
|
||||||
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --recursive https://github.com/uber/horovod.git /horovod_src || true
|
|
||||||
|
|
||||||
COPY bashrc /etc/bash.bashrc
|
|
||||||
RUN chmod a+rwx /etc/bash.bashrc
|
|
||||||
|
|
||||||
RUN python3 -m pip install --no-cache-dir jupyter matplotlib
|
|
||||||
# Pin ipykernel and nbformat; see https://github.com/ipython/ipykernel/issues/422
|
|
||||||
RUN python3 -m pip install --no-cache-dir jupyter_http_over_ws ipykernel==5.1.1 nbformat==4.4.0
|
|
||||||
RUN jupyter serverextension enable --py jupyter_http_over_ws
|
|
||||||
|
|
||||||
RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
|
|
||||||
RUN mkdir /.local && chmod a+rwx /.local
|
|
||||||
RUN apt-get install -y --no-install-recommends wget
|
|
||||||
# some examples require git to fetch dependencies
|
|
||||||
RUN apt-get install -y --no-install-recommends git
|
|
||||||
WORKDIR /tf/tensorflow-tutorials
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/regression.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/save_and_load.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/text_classification.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/text_classification_with_hub.ipynb
|
|
||||||
COPY readme-for-jupyter.md README.md
|
|
||||||
RUN apt-get autoremove -y && apt-get remove -y wget
|
|
||||||
WORKDIR /tf
|
|
||||||
EXPOSE 8888
|
|
||||||
|
|
||||||
RUN python3 -m ipykernel.kernelspec
|
|
||||||
|
|
||||||
CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
|
|
||||||
@ -1,4 +1,4 @@
|
|||||||
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
@ -19,11 +19,13 @@
|
|||||||
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
# for more information.
|
# for more information.
|
||||||
|
|
||||||
ARG UBUNTU_VERSION=18.04
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
FROM ubuntu:${UBUNTU_VERSION} AS base
|
FROM ubuntu:${UBUNTU_VERSION} AS base
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
build-essential \
|
build-essential \
|
||||||
curl \
|
curl \
|
||||||
git \
|
git \
|
||||||
@ -50,14 +52,13 @@ ENV CI_BUILD_PYTHON python
|
|||||||
ARG CACHE_STOP=1
|
ARG CACHE_STOP=1
|
||||||
# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
|
# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
|
||||||
ARG CHECKOUT_TF_SRC=0
|
ARG CHECKOUT_TF_SRC=0
|
||||||
# In case of Python 2.7+ we need to add passwd entries for user and group id
|
ARG TF_BRANCH=master
|
||||||
RUN chmod a+w /etc/passwd /etc/group
|
RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git --branch "${TF_BRANCH}" --single-branch /tensorflow_src || true
|
||||||
RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src || true
|
|
||||||
|
|
||||||
# See http://bugs.python.org/issue19846
|
# See http://bugs.python.org/issue19846
|
||||||
ENV LANG C.UTF-8
|
ENV LANG C.UTF-8
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
python3 \
|
python3 \
|
||||||
python3-pip
|
python3-pip
|
||||||
|
|
||||||
@ -68,78 +69,37 @@ RUN python3 -m pip --no-cache-dir install --upgrade \
|
|||||||
# Some TF tools expect a "python" binary
|
# Some TF tools expect a "python" binary
|
||||||
RUN ln -s $(which python3) /usr/local/bin/python
|
RUN ln -s $(which python3) /usr/local/bin/python
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
build-essential \
|
curl
|
||||||
curl \
|
|
||||||
git \
|
|
||||||
wget \
|
|
||||||
openjdk-8-jdk \
|
|
||||||
python3-dev \
|
|
||||||
virtualenv \
|
|
||||||
swig
|
|
||||||
|
|
||||||
RUN python3 -m pip --no-cache-dir install \
|
|
||||||
Pillow \
|
|
||||||
h5py \
|
|
||||||
keras_preprocessing \
|
|
||||||
matplotlib \
|
|
||||||
mock \
|
|
||||||
'numpy<1.19.0' \
|
|
||||||
scipy \
|
|
||||||
sklearn \
|
|
||||||
pandas \
|
|
||||||
future \
|
|
||||||
portpicker \
|
|
||||||
enum34
|
|
||||||
|
|
||||||
# Install bazel
|
# Install bazel
|
||||||
ARG BAZEL_VERSION=3.1.0
|
ARG BAZEL_VERSION=3.1.0
|
||||||
RUN mkdir /bazel && \
|
RUN mkdir /bazel && \
|
||||||
wget -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
curl -fSsL -o /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||||
wget -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
||||||
chmod +x /bazel/installer.sh && \
|
bash /bazel/installer.sh && \
|
||||||
/bazel/installer.sh && \
|
|
||||||
rm -f /bazel/installer.sh
|
rm -f /bazel/installer.sh
|
||||||
|
|
||||||
# install libnuma, openssh, wget
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
RUN ( apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
|
||||||
libnuma-dev \
|
|
||||||
openssh-server \
|
|
||||||
openssh-client \
|
|
||||||
wget && \
|
|
||||||
apt-get clean && \
|
|
||||||
rm -rf /var/lib/apt/lists/* ) || \
|
|
||||||
( yum -y update && yum -y install \
|
|
||||||
numactl-devel \
|
|
||||||
openssh-server \
|
|
||||||
openssh-clients \
|
|
||||||
wget && \
|
|
||||||
yum clean all ) || \
|
|
||||||
( echo "Unsupported Linux distribution. Aborting!" && exit 1 )
|
|
||||||
|
|
||||||
# Install Open MPI
|
# install libnuma, openssh, wget
|
||||||
# download realese version from official website as openmpi github master is not always stable
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
ARG OPENMPI_VERSION=openmpi-4.0.0
|
libopenmpi-dev \
|
||||||
ARG OPENMPI_DOWNLOAD_URL=https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.0.tar.gz
|
openmpi-bin \
|
||||||
RUN mkdir /tmp/openmpi && \
|
openmpi-common \
|
||||||
cd /tmp/openmpi && \
|
openssh-client \
|
||||||
wget ${OPENMPI_DOWNLOAD_URL} && \
|
openssh-server && \
|
||||||
tar zxf ${OPENMPI_VERSION}.tar.gz && \
|
apt-get clean && \
|
||||||
cd ${OPENMPI_VERSION} && \
|
rm -rf /var/lib/apt/lists/*
|
||||||
./configure --enable-orterun-prefix-by-default && \
|
|
||||||
make -j $(nproc) all && \
|
|
||||||
make install && \
|
|
||||||
ldconfig && \
|
|
||||||
rm -rf /tmp/openmpi
|
|
||||||
|
|
||||||
# Create a wrapper for OpenMPI to allow running as root by default
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
chmod a+x /usr/local/bin/mpirun
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
# Configure OpenMPI to run good defaults:
|
# Configure OpenMPI to run good defaults:
|
||||||
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
# Install OpenSSH for MPI to communicate between containers
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
RUN mkdir -p /var/run/sshd
|
RUN mkdir -p /var/run/sshd
|
||||||
@ -151,7 +111,22 @@ RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_confi
|
|||||||
|
|
||||||
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
||||||
ARG CHECKOUT_HOROVOD_SRC=0
|
ARG CHECKOUT_HOROVOD_SRC=0
|
||||||
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --recursive https://github.com/uber/horovod.git /horovod_src || true
|
ARG HOROVOD_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --branch "${HOROVOD_BRANCH}" --single-branch --recursive https://github.com/uber/horovod.git /horovod_src || true
|
||||||
|
|
||||||
COPY bashrc /etc/bash.bashrc
|
COPY bashrc /etc/bash.bashrc
|
||||||
RUN chmod a+rwx /etc/bash.bashrc
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter matplotlib
|
||||||
|
# Pin ipykernel and nbformat; see https://github.com/ipython/ipykernel/issues/422
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter_http_over_ws ipykernel==5.1.1 nbformat==4.4.0
|
||||||
|
RUN jupyter serverextension enable --py jupyter_http_over_ws
|
||||||
|
|
||||||
|
RUN mkdir -p /tf/ && chmod -R a+rwx /tf/
|
||||||
|
RUN mkdir /.local && chmod a+rwx /.local
|
||||||
|
WORKDIR /tf
|
||||||
|
EXPOSE 8888
|
||||||
|
|
||||||
|
RUN python3 -m ipykernel.kernelspec
|
||||||
|
|
||||||
|
CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
|
||||||
@ -0,0 +1,118 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} AS base
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
libcurl3-dev \
|
||||||
|
libfreetype6-dev \
|
||||||
|
libhdf5-serial-dev \
|
||||||
|
libzmq3-dev \
|
||||||
|
pkg-config \
|
||||||
|
rsync \
|
||||||
|
software-properties-common \
|
||||||
|
sudo \
|
||||||
|
unzip \
|
||||||
|
zip \
|
||||||
|
zlib1g-dev \
|
||||||
|
openjdk-8-jdk \
|
||||||
|
openjdk-8-jre-headless \
|
||||||
|
&& \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV CI_BUILD_PYTHON python
|
||||||
|
|
||||||
|
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
|
||||||
|
ARG CACHE_STOP=1
|
||||||
|
# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
|
||||||
|
ARG CHECKOUT_TF_SRC=0
|
||||||
|
ARG TF_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git --branch "${TF_BRANCH}" --single-branch /tensorflow_src || true
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
python3 \
|
||||||
|
python3-pip
|
||||||
|
|
||||||
|
RUN python3 -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -s $(which python3) /usr/local/bin/python
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl
|
||||||
|
|
||||||
|
# Install bazel
|
||||||
|
ARG BAZEL_VERSION=3.1.0
|
||||||
|
RUN mkdir /bazel && \
|
||||||
|
curl -fSsL -o /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||||
|
curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
||||||
|
bash /bazel/installer.sh && \
|
||||||
|
rm -f /bazel/installer.sh
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
||||||
|
ARG CHECKOUT_HOROVOD_SRC=0
|
||||||
|
ARG HOROVOD_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --branch "${HOROVOD_BRANCH}" --single-branch --recursive https://github.com/uber/horovod.git /horovod_src || true
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
@ -1,4 +1,4 @@
|
|||||||
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
@ -19,16 +19,14 @@
|
|||||||
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
# for more information.
|
# for more information.
|
||||||
|
|
||||||
ARG UBUNTU_VERSION=18.04
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
FROM ubuntu:${UBUNTU_VERSION} as base
|
FROM ubuntu:${UBUNTU_VERSION} as base
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y curl
|
|
||||||
|
|
||||||
# See http://bugs.python.org/issue19846
|
# See http://bugs.python.org/issue19846
|
||||||
ENV LANG C.UTF-8
|
ENV LANG C.UTF-8
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
python3 \
|
python3 \
|
||||||
python3-pip
|
python3-pip
|
||||||
|
|
||||||
@ -50,45 +48,26 @@ ARG TF_PACKAGE=tensorflow
|
|||||||
ARG TF_PACKAGE_VERSION=
|
ARG TF_PACKAGE_VERSION=
|
||||||
RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
|
RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
|
||||||
|
|
||||||
# install libnuma, openssh, wget
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
RUN ( apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
|
||||||
libnuma-dev \
|
|
||||||
openssh-server \
|
|
||||||
openssh-client \
|
|
||||||
wget && \
|
|
||||||
apt-get clean && \
|
|
||||||
rm -rf /var/lib/apt/lists/* ) || \
|
|
||||||
( yum -y update && yum -y install \
|
|
||||||
numactl-devel \
|
|
||||||
openssh-server \
|
|
||||||
openssh-clients \
|
|
||||||
wget && \
|
|
||||||
yum clean all ) || \
|
|
||||||
( echo "Unsupported Linux distribution. Aborting!" && exit 1 )
|
|
||||||
|
|
||||||
# Install Open MPI
|
# install libnuma, openssh, wget
|
||||||
# download realese version from official website as openmpi github master is not always stable
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
ARG OPENMPI_VERSION=openmpi-4.0.0
|
libopenmpi-dev \
|
||||||
ARG OPENMPI_DOWNLOAD_URL=https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.0.tar.gz
|
openmpi-bin \
|
||||||
RUN mkdir /tmp/openmpi && \
|
openmpi-common \
|
||||||
cd /tmp/openmpi && \
|
openssh-client \
|
||||||
wget ${OPENMPI_DOWNLOAD_URL} && \
|
openssh-server && \
|
||||||
tar zxf ${OPENMPI_VERSION}.tar.gz && \
|
apt-get clean && \
|
||||||
cd ${OPENMPI_VERSION} && \
|
rm -rf /var/lib/apt/lists/*
|
||||||
./configure --enable-orterun-prefix-by-default && \
|
|
||||||
make -j $(nproc) all && \
|
|
||||||
make install && \
|
|
||||||
ldconfig && \
|
|
||||||
rm -rf /tmp/openmpi
|
|
||||||
|
|
||||||
# Create a wrapper for OpenMPI to allow running as root by default
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
chmod a+x /usr/local/bin/mpirun
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
# Configure OpenMPI to run good defaults:
|
# Configure OpenMPI to run good defaults:
|
||||||
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
# Install OpenSSH for MPI to communicate between containers
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
RUN mkdir -p /var/run/sshd
|
RUN mkdir -p /var/run/sshd
|
||||||
@ -99,8 +78,26 @@ RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_confi
|
|||||||
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
# Install Horovod
|
# Install Horovod
|
||||||
ARG HOROVOD_VERSION=0.16.4
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
RUN python3 -m pip install --no-cache-dir horovod==${HOROVOD_VERSION}
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
software-properties-common
|
||||||
|
|
||||||
|
RUN add-apt-repository ppa:ubuntu-toolchain-r/test
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
python3-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 500 --slave /usr/bin/g++ g++ /usr/bin/g++-5 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
|
|
||||||
COPY bashrc /etc/bash.bashrc
|
COPY bashrc /etc/bash.bashrc
|
||||||
RUN chmod a+rwx /etc/bash.bashrc
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
@ -110,20 +107,8 @@ RUN python3 -m pip install --no-cache-dir jupyter matplotlib
|
|||||||
RUN python3 -m pip install --no-cache-dir jupyter_http_over_ws ipykernel==5.1.1 nbformat==4.4.0
|
RUN python3 -m pip install --no-cache-dir jupyter_http_over_ws ipykernel==5.1.1 nbformat==4.4.0
|
||||||
RUN jupyter serverextension enable --py jupyter_http_over_ws
|
RUN jupyter serverextension enable --py jupyter_http_over_ws
|
||||||
|
|
||||||
RUN mkdir -p /tf/tensorflow-tutorials && chmod -R a+rwx /tf/
|
RUN mkdir -p /tf/ && chmod -R a+rwx /tf/
|
||||||
RUN mkdir /.local && chmod a+rwx /.local
|
RUN mkdir /.local && chmod a+rwx /.local
|
||||||
RUN apt-get install -y --no-install-recommends wget
|
|
||||||
# some examples require git to fetch dependencies
|
|
||||||
RUN apt-get install -y --no-install-recommends git
|
|
||||||
WORKDIR /tf/tensorflow-tutorials
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/classification.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/overfit_and_underfit.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/regression.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/save_and_load.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/text_classification.ipynb
|
|
||||||
RUN wget https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/keras/text_classification_with_hub.ipynb
|
|
||||||
COPY readme-for-jupyter.md README.md
|
|
||||||
RUN apt-get autoremove -y && apt-get remove -y wget
|
|
||||||
WORKDIR /tf
|
WORKDIR /tf
|
||||||
EXPOSE 8888
|
EXPOSE 8888
|
||||||
|
|
||||||
@ -1,4 +1,4 @@
|
|||||||
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
# you may not use this file except in compliance with the License.
|
# you may not use this file except in compliance with the License.
|
||||||
@ -19,16 +19,14 @@
|
|||||||
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
# for more information.
|
# for more information.
|
||||||
|
|
||||||
ARG UBUNTU_VERSION=18.04
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
FROM ubuntu:${UBUNTU_VERSION} as base
|
FROM ubuntu:${UBUNTU_VERSION} as base
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y curl
|
|
||||||
|
|
||||||
# See http://bugs.python.org/issue19846
|
# See http://bugs.python.org/issue19846
|
||||||
ENV LANG C.UTF-8
|
ENV LANG C.UTF-8
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
python3 \
|
python3 \
|
||||||
python3-pip
|
python3-pip
|
||||||
|
|
||||||
@ -50,45 +48,26 @@ ARG TF_PACKAGE=tensorflow
|
|||||||
ARG TF_PACKAGE_VERSION=
|
ARG TF_PACKAGE_VERSION=
|
||||||
RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
|
RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
|
||||||
|
|
||||||
# install libnuma, openssh, wget
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
RUN ( apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
|
||||||
libnuma-dev \
|
|
||||||
openssh-server \
|
|
||||||
openssh-client \
|
|
||||||
wget && \
|
|
||||||
apt-get clean && \
|
|
||||||
rm -rf /var/lib/apt/lists/* ) || \
|
|
||||||
( yum -y update && yum -y install \
|
|
||||||
numactl-devel \
|
|
||||||
openssh-server \
|
|
||||||
openssh-clients \
|
|
||||||
wget && \
|
|
||||||
yum clean all ) || \
|
|
||||||
( echo "Unsupported Linux distribution. Aborting!" && exit 1 )
|
|
||||||
|
|
||||||
# Install Open MPI
|
# install libnuma, openssh, wget
|
||||||
# download realese version from official website as openmpi github master is not always stable
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
ARG OPENMPI_VERSION=openmpi-4.0.0
|
libopenmpi-dev \
|
||||||
ARG OPENMPI_DOWNLOAD_URL=https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.0.tar.gz
|
openmpi-bin \
|
||||||
RUN mkdir /tmp/openmpi && \
|
openmpi-common \
|
||||||
cd /tmp/openmpi && \
|
openssh-client \
|
||||||
wget ${OPENMPI_DOWNLOAD_URL} && \
|
openssh-server && \
|
||||||
tar zxf ${OPENMPI_VERSION}.tar.gz && \
|
apt-get clean && \
|
||||||
cd ${OPENMPI_VERSION} && \
|
rm -rf /var/lib/apt/lists/*
|
||||||
./configure --enable-orterun-prefix-by-default && \
|
|
||||||
make -j $(nproc) all && \
|
|
||||||
make install && \
|
|
||||||
ldconfig && \
|
|
||||||
rm -rf /tmp/openmpi
|
|
||||||
|
|
||||||
# Create a wrapper for OpenMPI to allow running as root by default
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
chmod a+x /usr/local/bin/mpirun
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
# Configure OpenMPI to run good defaults:
|
# Configure OpenMPI to run good defaults:
|
||||||
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
# Install OpenSSH for MPI to communicate between containers
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
RUN mkdir -p /var/run/sshd
|
RUN mkdir -p /var/run/sshd
|
||||||
@ -99,8 +78,26 @@ RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_confi
|
|||||||
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
# Install Horovod
|
# Install Horovod
|
||||||
ARG HOROVOD_VERSION=0.16.4
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
RUN python3 -m pip install --no-cache-dir horovod==${HOROVOD_VERSION}
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
software-properties-common
|
||||||
|
|
||||||
|
RUN add-apt-repository ppa:ubuntu-toolchain-r/test
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
python3-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 500 --slave /usr/bin/g++ g++ /usr/bin/g++-5 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
|
|
||||||
COPY bashrc /etc/bash.bashrc
|
COPY bashrc /etc/bash.bashrc
|
||||||
RUN chmod a+rwx /etc/bash.bashrc
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
@ -0,0 +1,132 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} AS base
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
libcurl3-dev \
|
||||||
|
libfreetype6-dev \
|
||||||
|
libhdf5-serial-dev \
|
||||||
|
libzmq3-dev \
|
||||||
|
pkg-config \
|
||||||
|
rsync \
|
||||||
|
software-properties-common \
|
||||||
|
sudo \
|
||||||
|
unzip \
|
||||||
|
zip \
|
||||||
|
zlib1g-dev \
|
||||||
|
openjdk-8-jdk \
|
||||||
|
openjdk-8-jre-headless \
|
||||||
|
&& \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV CI_BUILD_PYTHON python
|
||||||
|
|
||||||
|
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
|
||||||
|
ARG CACHE_STOP=1
|
||||||
|
# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
|
||||||
|
ARG CHECKOUT_TF_SRC=0
|
||||||
|
ARG TF_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git --branch "${TF_BRANCH}" --single-branch /tensorflow_src || true
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
python3 \
|
||||||
|
python3-pip
|
||||||
|
|
||||||
|
RUN python3 -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -s $(which python3) /usr/local/bin/python
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl
|
||||||
|
|
||||||
|
# Install bazel
|
||||||
|
ARG BAZEL_VERSION=3.1.0
|
||||||
|
RUN mkdir /bazel && \
|
||||||
|
curl -fSsL -o /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||||
|
curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
||||||
|
bash /bazel/installer.sh && \
|
||||||
|
rm -f /bazel/installer.sh
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
||||||
|
ARG CHECKOUT_HOROVOD_SRC=0
|
||||||
|
ARG HOROVOD_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --branch "${HOROVOD_BRANCH}" --single-branch --recursive https://github.com/uber/horovod.git /horovod_src || true
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter matplotlib
|
||||||
|
# Pin ipykernel and nbformat; see https://github.com/ipython/ipykernel/issues/422
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter_http_over_ws ipykernel==5.1.1 nbformat==4.4.0
|
||||||
|
RUN jupyter serverextension enable --py jupyter_http_over_ws
|
||||||
|
|
||||||
|
RUN mkdir -p /tf/ && chmod -R a+rwx /tf/
|
||||||
|
RUN mkdir /.local && chmod a+rwx /.local
|
||||||
|
WORKDIR /tf
|
||||||
|
EXPOSE 8888
|
||||||
|
|
||||||
|
RUN python3 -m ipykernel.kernelspec
|
||||||
|
|
||||||
|
CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
|
||||||
@ -0,0 +1,118 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} AS base
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
libcurl3-dev \
|
||||||
|
libfreetype6-dev \
|
||||||
|
libhdf5-serial-dev \
|
||||||
|
libzmq3-dev \
|
||||||
|
pkg-config \
|
||||||
|
rsync \
|
||||||
|
software-properties-common \
|
||||||
|
sudo \
|
||||||
|
unzip \
|
||||||
|
zip \
|
||||||
|
zlib1g-dev \
|
||||||
|
openjdk-8-jdk \
|
||||||
|
openjdk-8-jre-headless \
|
||||||
|
&& \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV CI_BUILD_PYTHON python
|
||||||
|
|
||||||
|
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
|
||||||
|
ARG CACHE_STOP=1
|
||||||
|
# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
|
||||||
|
ARG CHECKOUT_TF_SRC=0
|
||||||
|
ARG TF_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git --branch "${TF_BRANCH}" --single-branch /tensorflow_src || true
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
python3 \
|
||||||
|
python3-pip
|
||||||
|
|
||||||
|
RUN python3 -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -s $(which python3) /usr/local/bin/python
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl
|
||||||
|
|
||||||
|
# Install bazel
|
||||||
|
ARG BAZEL_VERSION=3.1.0
|
||||||
|
RUN mkdir /bazel && \
|
||||||
|
curl -fSsL -o /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||||
|
curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
||||||
|
bash /bazel/installer.sh && \
|
||||||
|
rm -f /bazel/installer.sh
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
||||||
|
ARG CHECKOUT_HOROVOD_SRC=0
|
||||||
|
ARG HOROVOD_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --branch "${HOROVOD_BRANCH}" --single-branch --recursive https://github.com/uber/horovod.git /horovod_src || true
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
@ -0,0 +1,112 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} as base
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
python3 \
|
||||||
|
python3-pip
|
||||||
|
|
||||||
|
RUN python3 -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -s $(which python3) /usr/local/bin/python
|
||||||
|
|
||||||
|
# Options:
|
||||||
|
# tensorflow
|
||||||
|
# tensorflow-gpu
|
||||||
|
# tf-nightly
|
||||||
|
# tf-nightly-gpu
|
||||||
|
# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
|
||||||
|
# Installs the latest version by default.
|
||||||
|
ARG TF_PACKAGE=tensorflow
|
||||||
|
ARG TF_PACKAGE_VERSION=
|
||||||
|
RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Install Horovod
|
||||||
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
python3-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter matplotlib
|
||||||
|
# Pin ipykernel and nbformat; see https://github.com/ipython/ipykernel/issues/422
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter_http_over_ws ipykernel==5.1.1 nbformat==4.4.0
|
||||||
|
RUN jupyter serverextension enable --py jupyter_http_over_ws
|
||||||
|
|
||||||
|
RUN mkdir -p /tf/ && chmod -R a+rwx /tf/
|
||||||
|
RUN mkdir /.local && chmod a+rwx /.local
|
||||||
|
WORKDIR /tf
|
||||||
|
EXPOSE 8888
|
||||||
|
|
||||||
|
RUN python3 -m ipykernel.kernelspec
|
||||||
|
|
||||||
|
CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
|
||||||
@ -0,0 +1,98 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} as base
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
python3 \
|
||||||
|
python3-pip
|
||||||
|
|
||||||
|
RUN python3 -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -s $(which python3) /usr/local/bin/python
|
||||||
|
|
||||||
|
# Options:
|
||||||
|
# tensorflow
|
||||||
|
# tensorflow-gpu
|
||||||
|
# tf-nightly
|
||||||
|
# tf-nightly-gpu
|
||||||
|
# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
|
||||||
|
# Installs the latest version by default.
|
||||||
|
ARG TF_PACKAGE=tensorflow
|
||||||
|
ARG TF_PACKAGE_VERSION=
|
||||||
|
RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Install Horovod
|
||||||
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
python3-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
@ -57,17 +57,27 @@ RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/t
|
|||||||
|
|
||||||
# See http://bugs.python.org/issue19846
|
# See http://bugs.python.org/issue19846
|
||||||
ENV LANG C.UTF-8
|
ENV LANG C.UTF-8
|
||||||
|
ARG PYTHON=python3
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
python3 \
|
curl \
|
||||||
python3-pip
|
software-properties-common
|
||||||
|
|
||||||
RUN python3 -m pip --no-cache-dir install --upgrade \
|
RUN add-apt-repository ppa:deadsnakes/ppa
|
||||||
|
|
||||||
|
RUN apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
${PYTHON}
|
||||||
|
|
||||||
|
RUN curl -fSsL https://bootstrap.pypa.io/get-pip.py | python3.7
|
||||||
|
RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \
|
||||||
pip \
|
pip \
|
||||||
setuptools
|
setuptools
|
||||||
|
|
||||||
# Some TF tools expect a "python" binary
|
# Some TF tools expect a "python" binary
|
||||||
RUN ln -s $(which python3) /usr/local/bin/python
|
RUN ln -sf $(which ${PYTHON}) /usr/local/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/local/bin/python3 && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python3
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
curl
|
curl
|
||||||
|
|||||||
@ -0,0 +1,142 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} AS base
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
libcurl3-dev \
|
||||||
|
libfreetype6-dev \
|
||||||
|
libhdf5-serial-dev \
|
||||||
|
libzmq3-dev \
|
||||||
|
pkg-config \
|
||||||
|
rsync \
|
||||||
|
software-properties-common \
|
||||||
|
sudo \
|
||||||
|
unzip \
|
||||||
|
zip \
|
||||||
|
zlib1g-dev \
|
||||||
|
openjdk-8-jdk \
|
||||||
|
openjdk-8-jre-headless \
|
||||||
|
&& \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV CI_BUILD_PYTHON python
|
||||||
|
|
||||||
|
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
|
||||||
|
ARG CACHE_STOP=1
|
||||||
|
# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
|
||||||
|
ARG CHECKOUT_TF_SRC=0
|
||||||
|
ARG TF_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git --branch "${TF_BRANCH}" --single-branch /tensorflow_src || true
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
ARG PYTHON=python3
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl \
|
||||||
|
software-properties-common
|
||||||
|
|
||||||
|
RUN add-apt-repository ppa:deadsnakes/ppa
|
||||||
|
|
||||||
|
RUN apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
${PYTHON}
|
||||||
|
|
||||||
|
RUN curl -fSsL https://bootstrap.pypa.io/get-pip.py | python3.7
|
||||||
|
RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -sf $(which ${PYTHON}) /usr/local/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/local/bin/python3 && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python3
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl
|
||||||
|
|
||||||
|
# Install bazel
|
||||||
|
ARG BAZEL_VERSION=3.1.0
|
||||||
|
RUN mkdir /bazel && \
|
||||||
|
curl -fSsL -o /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||||
|
curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
||||||
|
bash /bazel/installer.sh && \
|
||||||
|
rm -f /bazel/installer.sh
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
||||||
|
ARG CHECKOUT_HOROVOD_SRC=0
|
||||||
|
ARG HOROVOD_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --branch "${HOROVOD_BRANCH}" --single-branch --recursive https://github.com/uber/horovod.git /horovod_src || true
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter matplotlib
|
||||||
|
# Pin ipykernel and nbformat; see https://github.com/ipython/ipykernel/issues/422
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter_http_over_ws ipykernel==5.1.1 nbformat==4.4.0
|
||||||
|
RUN jupyter serverextension enable --py jupyter_http_over_ws
|
||||||
|
|
||||||
|
RUN mkdir -p /tf/ && chmod -R a+rwx /tf/
|
||||||
|
RUN mkdir /.local && chmod a+rwx /.local
|
||||||
|
WORKDIR /tf
|
||||||
|
EXPOSE 8888
|
||||||
|
|
||||||
|
RUN python3 -m ipykernel.kernelspec
|
||||||
|
|
||||||
|
CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
|
||||||
@ -0,0 +1,128 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} AS base
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
libcurl3-dev \
|
||||||
|
libfreetype6-dev \
|
||||||
|
libhdf5-serial-dev \
|
||||||
|
libzmq3-dev \
|
||||||
|
pkg-config \
|
||||||
|
rsync \
|
||||||
|
software-properties-common \
|
||||||
|
sudo \
|
||||||
|
unzip \
|
||||||
|
zip \
|
||||||
|
zlib1g-dev \
|
||||||
|
openjdk-8-jdk \
|
||||||
|
openjdk-8-jre-headless \
|
||||||
|
&& \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
ENV CI_BUILD_PYTHON python
|
||||||
|
|
||||||
|
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version
|
||||||
|
ARG CACHE_STOP=1
|
||||||
|
# Check out TensorFlow source code if --build-arg CHECKOUT_TF_SRC=1
|
||||||
|
ARG CHECKOUT_TF_SRC=0
|
||||||
|
ARG TF_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/tensorflow.git --branch "${TF_BRANCH}" --single-branch /tensorflow_src || true
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
ARG PYTHON=python3
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl \
|
||||||
|
software-properties-common
|
||||||
|
|
||||||
|
RUN add-apt-repository ppa:deadsnakes/ppa
|
||||||
|
|
||||||
|
RUN apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
${PYTHON}
|
||||||
|
|
||||||
|
RUN curl -fSsL https://bootstrap.pypa.io/get-pip.py | python3.7
|
||||||
|
RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -sf $(which ${PYTHON}) /usr/local/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/local/bin/python3 && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python3
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl
|
||||||
|
|
||||||
|
# Install bazel
|
||||||
|
ARG BAZEL_VERSION=3.1.0
|
||||||
|
RUN mkdir /bazel && \
|
||||||
|
curl -fSsL -o /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
|
||||||
|
curl -fSsL -o /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
|
||||||
|
bash /bazel/installer.sh && \
|
||||||
|
rm -f /bazel/installer.sh
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
||||||
|
ARG CHECKOUT_HOROVOD_SRC=0
|
||||||
|
ARG HOROVOD_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --branch "${HOROVOD_BRANCH}" --single-branch --recursive https://github.com/uber/horovod.git /horovod_src || true
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
@ -57,17 +57,27 @@ RUN test "${CHECKOUT_TF_SRC}" -eq 1 && git clone https://github.com/tensorflow/t
|
|||||||
|
|
||||||
# See http://bugs.python.org/issue19846
|
# See http://bugs.python.org/issue19846
|
||||||
ENV LANG C.UTF-8
|
ENV LANG C.UTF-8
|
||||||
|
ARG PYTHON=python3
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
python3 \
|
curl \
|
||||||
python3-pip
|
software-properties-common
|
||||||
|
|
||||||
RUN python3 -m pip --no-cache-dir install --upgrade \
|
RUN add-apt-repository ppa:deadsnakes/ppa
|
||||||
|
|
||||||
|
RUN apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
${PYTHON}
|
||||||
|
|
||||||
|
RUN curl -fSsL https://bootstrap.pypa.io/get-pip.py | python3.7
|
||||||
|
RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \
|
||||||
pip \
|
pip \
|
||||||
setuptools
|
setuptools
|
||||||
|
|
||||||
# Some TF tools expect a "python" binary
|
# Some TF tools expect a "python" binary
|
||||||
RUN ln -s $(which python3) /usr/local/bin/python
|
RUN ln -sf $(which ${PYTHON}) /usr/local/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/local/bin/python3 && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python3
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
curl
|
curl
|
||||||
|
|||||||
@ -0,0 +1,122 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} as base
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
ARG PYTHON=python3
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl \
|
||||||
|
software-properties-common
|
||||||
|
|
||||||
|
RUN add-apt-repository ppa:deadsnakes/ppa
|
||||||
|
|
||||||
|
RUN apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
${PYTHON}
|
||||||
|
|
||||||
|
RUN curl -fSsL https://bootstrap.pypa.io/get-pip.py | python3.7
|
||||||
|
RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -sf $(which ${PYTHON}) /usr/local/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/local/bin/python3 && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python3
|
||||||
|
|
||||||
|
# Options:
|
||||||
|
# tensorflow
|
||||||
|
# tensorflow-gpu
|
||||||
|
# tf-nightly
|
||||||
|
# tf-nightly-gpu
|
||||||
|
# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
|
||||||
|
# Installs the latest version by default.
|
||||||
|
ARG TF_PACKAGE=tensorflow
|
||||||
|
ARG TF_PACKAGE_VERSION=
|
||||||
|
RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Install Horovod
|
||||||
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
${PYTHON}-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter matplotlib
|
||||||
|
# Pin ipykernel and nbformat; see https://github.com/ipython/ipykernel/issues/422
|
||||||
|
RUN python3 -m pip install --no-cache-dir jupyter_http_over_ws ipykernel==5.1.1 nbformat==4.4.0
|
||||||
|
RUN jupyter serverextension enable --py jupyter_http_over_ws
|
||||||
|
|
||||||
|
RUN mkdir -p /tf/ && chmod -R a+rwx /tf/
|
||||||
|
RUN mkdir /.local && chmod a+rwx /.local
|
||||||
|
WORKDIR /tf
|
||||||
|
EXPOSE 8888
|
||||||
|
|
||||||
|
RUN python3 -m ipykernel.kernelspec
|
||||||
|
|
||||||
|
CMD ["bash", "-c", "source /etc/bash.bashrc && jupyter notebook --notebook-dir=/tf --ip 0.0.0.0 --no-browser --allow-root"]
|
||||||
@ -0,0 +1,108 @@
|
|||||||
|
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ============================================================================
|
||||||
|
#
|
||||||
|
# THIS IS A GENERATED DOCKERFILE.
|
||||||
|
#
|
||||||
|
# This file was assembled from multiple pieces, whose use is documented
|
||||||
|
# throughout. Please refer to the TensorFlow dockerfiles documentation
|
||||||
|
# for more information.
|
||||||
|
|
||||||
|
ARG UBUNTU_VERSION=20.04
|
||||||
|
|
||||||
|
FROM ubuntu:${UBUNTU_VERSION} as base
|
||||||
|
|
||||||
|
# See http://bugs.python.org/issue19846
|
||||||
|
ENV LANG C.UTF-8
|
||||||
|
ARG PYTHON=python3
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
curl \
|
||||||
|
software-properties-common
|
||||||
|
|
||||||
|
RUN add-apt-repository ppa:deadsnakes/ppa
|
||||||
|
|
||||||
|
RUN apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
${PYTHON}
|
||||||
|
|
||||||
|
RUN curl -fSsL https://bootstrap.pypa.io/get-pip.py | python3.7
|
||||||
|
RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \
|
||||||
|
pip \
|
||||||
|
setuptools
|
||||||
|
|
||||||
|
# Some TF tools expect a "python" binary
|
||||||
|
RUN ln -sf $(which ${PYTHON}) /usr/local/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/local/bin/python3 && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python && \
|
||||||
|
ln -sf $(which ${PYTHON}) /usr/bin/python3
|
||||||
|
|
||||||
|
# Options:
|
||||||
|
# tensorflow
|
||||||
|
# tensorflow-gpu
|
||||||
|
# tf-nightly
|
||||||
|
# tf-nightly-gpu
|
||||||
|
# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
|
||||||
|
# Installs the latest version by default.
|
||||||
|
ARG TF_PACKAGE=tensorflow
|
||||||
|
ARG TF_PACKAGE_VERSION=
|
||||||
|
RUN python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
|
||||||
|
|
||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
|
|
||||||
|
# Install Horovod
|
||||||
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
${PYTHON}-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
|
|
||||||
|
COPY bashrc /etc/bash.bashrc
|
||||||
|
RUN chmod a+rwx /etc/bash.bashrc
|
||||||
@ -1,3 +0,0 @@
|
|||||||
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
|
||||||
ARG CHECKOUT_HOROVOD_SRC=0
|
|
||||||
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --recursive https://github.com/uber/horovod.git /horovod_src || true
|
|
||||||
@ -1,3 +0,0 @@
|
|||||||
# Install Horovod
|
|
||||||
ARG HOROVOD_VERSION=0.16.4
|
|
||||||
RUN python3 -m pip install --no-cache-dir horovod==${HOROVOD_VERSION}
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
# install libnuma, openssh, wget
|
|
||||||
RUN ( apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
|
||||||
libnuma-dev \
|
|
||||||
openssh-server \
|
|
||||||
openssh-client \
|
|
||||||
wget && \
|
|
||||||
apt-get clean && \
|
|
||||||
rm -rf /var/lib/apt/lists/* ) || \
|
|
||||||
( yum -y update && yum -y install \
|
|
||||||
numactl-devel \
|
|
||||||
openssh-server \
|
|
||||||
openssh-clients \
|
|
||||||
wget && \
|
|
||||||
yum clean all ) || \
|
|
||||||
( echo "Unsupported Linux distribution. Aborting!" && exit 1 )
|
|
||||||
|
|
||||||
# Install Open MPI
|
|
||||||
# download realese version from official website as openmpi github master is not always stable
|
|
||||||
ARG OPENMPI_VERSION=openmpi-4.0.0
|
|
||||||
ARG OPENMPI_DOWNLOAD_URL=https://www.open-mpi.org/software/ompi/v4.0/downloads/openmpi-4.0.0.tar.gz
|
|
||||||
RUN mkdir /tmp/openmpi && \
|
|
||||||
cd /tmp/openmpi && \
|
|
||||||
wget ${OPENMPI_DOWNLOAD_URL} && \
|
|
||||||
tar zxf ${OPENMPI_VERSION}.tar.gz && \
|
|
||||||
cd ${OPENMPI_VERSION} && \
|
|
||||||
./configure --enable-orterun-prefix-by-default && \
|
|
||||||
make -j $(nproc) all && \
|
|
||||||
make install && \
|
|
||||||
ldconfig && \
|
|
||||||
rm -rf /tmp/openmpi
|
|
||||||
|
|
||||||
# Create a wrapper for OpenMPI to allow running as root by default
|
|
||||||
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
|
|
||||||
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
|
|
||||||
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
|
|
||||||
chmod a+x /usr/local/bin/mpirun
|
|
||||||
|
|
||||||
# Configure OpenMPI to run good defaults:
|
|
||||||
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
|
|
||||||
|
|
||||||
# Install OpenSSH for MPI to communicate between containers
|
|
||||||
RUN mkdir -p /var/run/sshd
|
|
||||||
|
|
||||||
# Allow OpenSSH to talk to containers without asking for confirmation
|
|
||||||
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
|
||||||
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
|
||||||
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
|
||||||
@ -0,0 +1,21 @@
|
|||||||
|
# Install Horovod
|
||||||
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
software-properties-common
|
||||||
|
|
||||||
|
RUN add-apt-repository ppa:ubuntu-toolchain-r/test
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
python3-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 500 --slave /usr/bin/g++ g++ /usr/bin/g++-5 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
@ -0,0 +1,16 @@
|
|||||||
|
# Install Horovod
|
||||||
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
${PYTHON}-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
# Check out horovod source code if --build-arg CHECKOUT_HOROVOD_SRC=1
|
||||||
|
ARG CHECKOUT_HOROVOD_SRC=0
|
||||||
|
ARG HOROVOD_BRANCH=master
|
||||||
|
RUN test "${CHECKOUT_HOROVOD_SRC}" -eq 1 && git clone --branch "${HOROVOD_BRANCH}" --single-branch --recursive https://github.com/uber/horovod.git /horovod_src || true
|
||||||
@ -0,0 +1,16 @@
|
|||||||
|
# Install Horovod
|
||||||
|
ARG HOROVOD_WITHOUT_PYTORCH=1
|
||||||
|
ARG HOROVOD_WITHOUT_MXNET=1
|
||||||
|
ARG HOROVOD_WITH_TENSORFLOW=1
|
||||||
|
ARG HOROVOD_VERSION=
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
build-essential \
|
||||||
|
g++-8 \
|
||||||
|
gcc-8 \
|
||||||
|
python3-dev
|
||||||
|
|
||||||
|
RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7 && \
|
||||||
|
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 800 --slave /usr/bin/g++ g++ /usr/bin/g++-8
|
||||||
|
|
||||||
|
RUN python3 -m pip install --no-cache-dir horovod${HOROVOD_VERSION:+==${HOROVOD_VERSION}}
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
ARG DEBIAN_FRONTEND="noninteractive"
|
||||||
|
|
||||||
|
# install libnuma, openssh, wget
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
|
||||||
|
libopenmpi-dev \
|
||||||
|
openmpi-bin \
|
||||||
|
openmpi-common \
|
||||||
|
openssh-client \
|
||||||
|
openssh-server && \
|
||||||
|
apt-get clean && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create a wrapper for OpenMPI to allow running as root by default
|
||||||
|
RUN mv /usr/bin/mpirun /usr/bin/mpirun.real && \
|
||||||
|
echo '#!/bin/bash' > /usr/bin/mpirun && \
|
||||||
|
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/bin/mpirun && \
|
||||||
|
chmod a+x /usr/bin/mpirun
|
||||||
|
|
||||||
|
# Configure OpenMPI to run good defaults:
|
||||||
|
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /etc/openmpi/openmpi-mca-params.conf
|
||||||
|
|
||||||
|
# Install OpenSSH for MPI to communicate between containers
|
||||||
|
RUN mkdir -p /var/run/sshd
|
||||||
|
|
||||||
|
# Allow OpenSSH to talk to containers without asking for confirmation
|
||||||
|
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
|
||||||
|
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
|
||||||
|
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
|
||||||
@ -32,7 +32,6 @@ releases:
|
|||||||
tag_specs:
|
tag_specs:
|
||||||
- "{nightly}{jupyter}"
|
- "{nightly}{jupyter}"
|
||||||
- "{_TAG_PREFIX}{ubuntu-devel}"
|
- "{_TAG_PREFIX}{ubuntu-devel}"
|
||||||
|
|
||||||
# Built per-release and pushed to tensorflow/tensorflow
|
# Built per-release and pushed to tensorflow/tensorflow
|
||||||
# --arg _TAG_PREFIX=<val> should be set to "1.11" (for example) or "latest".
|
# --arg _TAG_PREFIX=<val> should be set to "1.11" (for example) or "latest".
|
||||||
versioned:
|
versioned:
|
||||||
@ -44,6 +43,10 @@ releases:
|
|||||||
- "{_TAG_PREFIX}{ubuntu-onednn}{onednn-jupyter}"
|
- "{_TAG_PREFIX}{ubuntu-onednn}{onednn-jupyter}"
|
||||||
- "{_TAG_PREFIX}{ubuntu-devel-onednn}"
|
- "{_TAG_PREFIX}{ubuntu-devel-onednn}"
|
||||||
- "{_TAG_PREFIX}{ubuntu-devel-onednn}{onednn-jupyter}"
|
- "{_TAG_PREFIX}{ubuntu-devel-onednn}{onednn-jupyter}"
|
||||||
|
- "{_TAG_PREFIX}{ubuntu-onednn-mpi-horovod}"
|
||||||
|
- "{_TAG_PREFIX}{ubuntu-onednn-mpi-horovod}{onednn-jupyter}"
|
||||||
|
- "{_TAG_PREFIX}{ubuntu-devel-onednn-mpi-horovod}"
|
||||||
|
- "{_TAG_PREFIX}{ubuntu-devel-onednn-mpi-horovod}{onednn-jupyter}"
|
||||||
|
|
||||||
# Dockerfiles stored in the TF repo; not pushed anywhere
|
# Dockerfiles stored in the TF repo; not pushed anywhere
|
||||||
dockerfiles:
|
dockerfiles:
|
||||||
@ -54,12 +57,14 @@ releases:
|
|||||||
- "{ubuntu-devel}{jupyter}"
|
- "{ubuntu-devel}{jupyter}"
|
||||||
- "{ubuntu-ppc64le}{jupyter}"
|
- "{ubuntu-ppc64le}{jupyter}"
|
||||||
- "{ubuntu-devel-ppc64le}{jupyter}"
|
- "{ubuntu-devel-ppc64le}{jupyter}"
|
||||||
- "{ubuntu-horovod}{jupyter}"
|
|
||||||
- "{ubuntu-devel-horovod}{jupyter}"
|
|
||||||
- "{ubuntu-onednn}"
|
- "{ubuntu-onednn}"
|
||||||
- "{ubuntu-onednn}{onednn-jupyter}"
|
- "{ubuntu-onednn}{onednn-jupyter}"
|
||||||
- "{ubuntu-devel-onednn}"
|
- "{ubuntu-devel-onednn}"
|
||||||
- "{ubuntu-devel-onednn}{onednn-jupyter}"
|
- "{ubuntu-devel-onednn}{onednn-jupyter}"
|
||||||
|
- "{ubuntu-onednn-mpi-horovod}"
|
||||||
|
- "{ubuntu-devel-onednn-mpi-horovod}"
|
||||||
|
- "{ubuntu-onednn-mpi-horovod}{onednn-jupyter}"
|
||||||
|
- "{ubuntu-devel-onednn-mpi-horovod}{onednn-jupyter}"
|
||||||
- "{ubuntu-devel-arm64v8}{jupyter}"
|
- "{ubuntu-devel-arm64v8}{jupyter}"
|
||||||
|
|
||||||
slice_sets:
|
slice_sets:
|
||||||
@ -142,40 +147,209 @@ slice_sets:
|
|||||||
- UBUNTU_VERSION=18.04
|
- UBUNTU_VERSION=18.04
|
||||||
- CHECKOUT_TF_SRC=1
|
- CHECKOUT_TF_SRC=1
|
||||||
|
|
||||||
ubuntu-horovod:
|
ubuntu-onednn:
|
||||||
- add_to_name: "-horovod"
|
- add_to_name: "-16.04-onednn"
|
||||||
dockerfile_exclusive_name: "horovod"
|
dockerfile_exclusive_name: "ubuntu-16.04-onednn"
|
||||||
dockerfile_subdirectory: "mkl_horovod"
|
dockerfile_subdirectory: "onednn"
|
||||||
partials:
|
partials:
|
||||||
- ubuntu/version
|
- onednn/ubuntu/version
|
||||||
- ubuntu/cpu
|
- onednn/ubuntu/cpu
|
||||||
- ubuntu/python
|
- onednn/ubuntu/python
|
||||||
- tensorflow
|
- tensorflow
|
||||||
- mkl_horovod/mpi
|
|
||||||
- mkl_horovod/horovod
|
|
||||||
- shell
|
- shell
|
||||||
tests:
|
tests:
|
||||||
- import-mkl-horovod.sh
|
- import-onednn.sh
|
||||||
args:
|
args:
|
||||||
- TF_PACKAGE=intel-tensorflow
|
- TF_PACKAGE=intel-tensorflow
|
||||||
|
- UBUNTU_VERSION=16.04
|
||||||
ubuntu-devel-horovod:
|
- add_to_name: "-18.04-onednn"
|
||||||
- add_to_name: "devel-horovod"
|
dockerfile_exclusive_name: "ubuntu-18.04-onednn"
|
||||||
dockerfile_exclusive_name: "devel-horovod"
|
dockerfile_subdirectory: "onednn"
|
||||||
dockerfile_subdirectory: "mkl_horovod"
|
|
||||||
partials:
|
partials:
|
||||||
- ubuntu/version
|
- onednn/ubuntu/version
|
||||||
- ubuntu/devel-cpu
|
- onednn/ubuntu/cpu
|
||||||
- ubuntu/python
|
- onednn/ubuntu/python
|
||||||
- ubuntu/bazel
|
- tensorflow
|
||||||
- mkl_horovod/mpi
|
|
||||||
- mkl_horovod/devel-horovod
|
|
||||||
- shell
|
- shell
|
||||||
tests:
|
tests:
|
||||||
- build-mkl-horovod.sh
|
- import-onednn.sh
|
||||||
args:
|
args:
|
||||||
|
- TF_PACKAGE=intel-tensorflow
|
||||||
|
- UBUNTU_VERSION=18.04
|
||||||
|
- add_to_name: "-20.04-onednn"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-20.04-onednn"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/cpu
|
||||||
|
- onednn/ubuntu/python3
|
||||||
|
- tensorflow
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- import-onednn.sh
|
||||||
|
args:
|
||||||
|
- TF_PACKAGE=intel-tensorflow
|
||||||
|
- UBUNTU_VERSION=20.04
|
||||||
|
- PYTHON=python3.7
|
||||||
|
|
||||||
|
ubuntu-devel-onednn:
|
||||||
|
- add_to_name: "-16.04-devel-onednn"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-16.04-devel-onednn"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/devel
|
||||||
|
- onednn/ubuntu/python
|
||||||
|
- onednn/ubuntu/bazel
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- ""
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=16.04
|
||||||
|
- CHECKOUT_TF_SRC=1
|
||||||
|
- TF_BRANCH=master
|
||||||
|
- add_to_name: "-18.04-devel-onednn"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-18.04-devel-onednn"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/devel
|
||||||
|
- onednn/ubuntu/python
|
||||||
|
- onednn/ubuntu/bazel
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- ""
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=18.04
|
||||||
|
- CHECKOUT_TF_SRC=1
|
||||||
|
- TF_BRANCH=master
|
||||||
|
- add_to_name: "-20.04-devel-onednn"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-20.04-devel-onednn"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/devel
|
||||||
|
- onednn/ubuntu/python3
|
||||||
|
- onednn/ubuntu/bazel
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- ""
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=20.04
|
||||||
|
- PYTHON=python3.7
|
||||||
|
- CHECKOUT_TF_SRC=1
|
||||||
|
- TF_BRANCH=master
|
||||||
|
|
||||||
|
ubuntu-onednn-mpi-horovod:
|
||||||
|
- add_to_name: "-16.04-onednn-mpi-horovod"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-16.04-onednn-mpi-horovod"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/cpu
|
||||||
|
- onednn/ubuntu/python
|
||||||
|
- tensorflow
|
||||||
|
- onednn/ubuntu/mpi
|
||||||
|
- onednn/ubuntu/1604-horovod
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- import-onednn-horovod.sh
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=16.04
|
||||||
|
- DEBIAN_FRONTEND="noninteractive"
|
||||||
|
- TF_PACKAGE=intel-tensorflow
|
||||||
|
- add_to_name: "-18.04-onednn-mpi-horovod"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-18.04-onednn-mpi-horovod"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/cpu
|
||||||
|
- onednn/ubuntu/python
|
||||||
|
- tensorflow
|
||||||
|
- onednn/ubuntu/mpi
|
||||||
|
- onednn/ubuntu/horovod
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- import-onednn-horovod.sh
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=18.04
|
||||||
|
- DEBIAN_FRONTEND="noninteractive"
|
||||||
|
- TF_PACKAGE=intel-tensorflow
|
||||||
|
- add_to_name: "-20.04-onednn-mpi-horovod"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-20.04-onednn-mpi-horovod"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/cpu
|
||||||
|
- onednn/ubuntu/python3
|
||||||
|
- tensorflow
|
||||||
|
- onednn/ubuntu/mpi
|
||||||
|
- onednn/ubuntu/2004-horovod
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- import-onednn-horovod.sh
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=20.04
|
||||||
|
- PYTHON=python3.7
|
||||||
|
- DEBIAN_FRONTEND="noninteractive"
|
||||||
|
- TF_PACKAGE=intel-tensorflow
|
||||||
|
|
||||||
|
ubuntu-devel-onednn-mpi-horovod:
|
||||||
|
- add_to_name: "-16.04-onednn-devel-mpi-horovod"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-16.04-devel-onednn-mpi-horovod"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/devel
|
||||||
|
- onednn/ubuntu/python
|
||||||
|
- onednn/ubuntu/bazel
|
||||||
|
- onednn/ubuntu/mpi
|
||||||
|
- onednn/ubuntu/devel-horovod
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- ""
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=16.04
|
||||||
- CHECKOUT_TF_SRC=1
|
- CHECKOUT_TF_SRC=1
|
||||||
- CHECKOUT_HOROVOD_SRC=1
|
- CHECKOUT_HOROVOD_SRC=1
|
||||||
|
- HOROVOD_BRANCH=master
|
||||||
|
- add_to_name: "-18.04-onednn-devel-mpi-horovod"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-18.04-devel-onednn-mpi-horovod"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/devel
|
||||||
|
- onednn/ubuntu/python
|
||||||
|
- onednn/ubuntu/bazel
|
||||||
|
- onednn/ubuntu/mpi
|
||||||
|
- onednn/ubuntu/devel-horovod
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- ""
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=18.04
|
||||||
|
- CHECKOUT_TF_SRC=1
|
||||||
|
- CHECKOUT_HOROVOD_SRC=1
|
||||||
|
- HOROVOD_BRANCH=master
|
||||||
|
- add_to_name: "-20.04-onednn-devel-mpi-horovod"
|
||||||
|
dockerfile_exclusive_name: "ubuntu-20.04-devel-onednn-mpi-horovod"
|
||||||
|
dockerfile_subdirectory: "onednn"
|
||||||
|
partials:
|
||||||
|
- onednn/ubuntu/version
|
||||||
|
- onednn/ubuntu/devel
|
||||||
|
- onednn/ubuntu/python3
|
||||||
|
- onednn/ubuntu/bazel
|
||||||
|
- onednn/ubuntu/mpi
|
||||||
|
- onednn/ubuntu/devel-horovod
|
||||||
|
- shell
|
||||||
|
tests:
|
||||||
|
- ""
|
||||||
|
args:
|
||||||
|
- UBUNTU_VERSION=20.04
|
||||||
|
- PYTHON=python3.7
|
||||||
|
- CHECKOUT_TF_SRC=1
|
||||||
|
- CHECKOUT_HOROVOD_SRC=1
|
||||||
|
- HOROVOD_BRANCH=master
|
||||||
|
|
||||||
|
|
||||||
ubuntu-onednn:
|
ubuntu-onednn:
|
||||||
|
|||||||
@ -15,4 +15,12 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
python -c 'from tensorflow.python import pywrap_tensorflow; pywrap_tensorflow.IsMklEnabled() or exit(1); import horovod.tensorflow as hvd'
|
{ # try
|
||||||
|
echo `python -c 'from tensorflow.python import _pywrap_util_port; print(_pywrap_util_port.IsMklEnabled()); import horovod.tensorflow as hvd'`
|
||||||
|
echo "PASS: Horovod with MKL is enabled"
|
||||||
|
} || { # catch
|
||||||
|
echo `python -c 'from tensorflow.python import pywrap_tensorflow; print(pywrap_tensorflow.IsMklEnabled()); import horovod.tensorflow as hvd'`
|
||||||
|
echo "PASS: Horovod with Old MKL is detected"
|
||||||
|
} || { # finally
|
||||||
|
die "FAIL: Horovod with MKL is not enabled"
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user