experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #35281 from angerson:master

Browse Source 
PiperOrigin-RevId: 286463433
Change-Id: Ia14d297859bdc44dfa3534d1ea3ae648ab203a7c
This commit is contained in:
TensorFlower Gardener 2019-12-19 14:39:40 -08:00
commit 13dd95a2ae
12 changed files with 104 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tensorflow/tools/dockerfiles/dockerfiles/devel-cpu-jupyter.Dockerfile
View File

@ -34,7 +34,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config \
rsync \
software-properties-common \
sudo \
sudo \
unzip \
zip \
zlib1g-dev \

3
tensorflow/tools/dockerfiles/dockerfiles/devel-cpu.Dockerfile
View File

@ -34,7 +34,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config \
rsync \
software-properties-common \
sudo \
sudo \
unzip \
zip \
zlib1g-dev \
@ -108,3 +108,4 @@ RUN mkdir /bazel && \
rm -f /bazel/installer.sh
COPY bashrc /etc/bash.bashrc
RUN chmod a+rwx /etc/bash.bashrc

20
tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile
View File

@ -22,22 +22,25 @@
ARG UBUNTU_VERSION=18.04
ARG ARCH=
ARG CUDA=10.0
ARG CUDA=10.1
FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
# ARCH and CUDA are specified again because the FROM directive resets ARGs
# (but their default value is retained if set previously)
ARG ARCH
ARG CUDA
ARG CUDNN=7.6.2.24-1
ARG CUDNN=7.6.4.38-1
ARG CUDNN_MAJOR_VERSION=7
ARG LIB_DIR_PREFIX=x86_64
ARG LIBNVINFER=6.0.1-1
ARG LIBNVINFER_MAJOR_VERSION=6
# Needed for string substitution
SHELL ["/bin/bash", "-c"]
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
cuda-command-line-tools-${CUDA/./-} \
cuda-cublas-dev-${CUDA/./-} \
libcublas10 \
libcublas-dev \
cuda-cudart-dev-${CUDA/./-} \
cuda-cufft-dev-${CUDA/./-} \
cuda-curand-dev-${CUDA/./-} \
@ -61,18 +64,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
# Install TensorRT if not building for PowerPC
RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \
apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda${CUDA} \
libnvinfer-dev=5.1.5-1+cuda${CUDA} \
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*; }
# Configure the build for our CUDA configuration.
ENV CI_BUILD_PYTHON python
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/usr/include/x64_64-linux-gnu:$LD_LIBRARY_PATH
ENV TF_NEED_CUDA 1
ENV TF_NEED_TENSORRT 1
ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
ENV TF_CUDA_VERSION=${CUDA}
ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION}
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version

20
tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile
View File

@ -22,22 +22,25 @@
ARG UBUNTU_VERSION=18.04
ARG ARCH=
ARG CUDA=10.0
ARG CUDA=10.1
FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
# ARCH and CUDA are specified again because the FROM directive resets ARGs
# (but their default value is retained if set previously)
ARG ARCH
ARG CUDA
ARG CUDNN=7.6.2.24-1
ARG CUDNN=7.6.4.38-1
ARG CUDNN_MAJOR_VERSION=7
ARG LIB_DIR_PREFIX=x86_64
ARG LIBNVINFER=6.0.1-1
ARG LIBNVINFER_MAJOR_VERSION=6
# Needed for string substitution
SHELL ["/bin/bash", "-c"]
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
cuda-command-line-tools-${CUDA/./-} \
cuda-cublas-dev-${CUDA/./-} \
libcublas10 \
libcublas-dev \
cuda-cudart-dev-${CUDA/./-} \
cuda-cufft-dev-${CUDA/./-} \
cuda-curand-dev-${CUDA/./-} \
@ -61,18 +64,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
# Install TensorRT if not building for PowerPC
RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \
apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda${CUDA} \
libnvinfer-dev=5.1.5-1+cuda${CUDA} \
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*; }
# Configure the build for our CUDA configuration.
ENV CI_BUILD_PYTHON python
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/usr/include/x64_64-linux-gnu:$LD_LIBRARY_PATH
ENV TF_NEED_CUDA 1
ENV TF_NEED_TENSORRT 1
ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
ENV TF_CUDA_VERSION=${CUDA}
ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION}
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version

2
tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le-jupyter.Dockerfile
View File

@ -34,7 +34,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config \
rsync \
software-properties-common \
sudo \
sudo \
unzip \
zip \
zlib1g-dev \

2
tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-cpu-ppc64le.Dockerfile
View File

@ -34,7 +34,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config \
rsync \
software-properties-common \
sudo \
sudo \
unzip \
zip \
zlib1g-dev \

20
tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile
View File

@ -22,22 +22,25 @@
ARG UBUNTU_VERSION=18.04
ARG ARCH=
ARG CUDA=10.0
ARG CUDA=10.1
FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
# ARCH and CUDA are specified again because the FROM directive resets ARGs
# (but their default value is retained if set previously)
ARG ARCH
ARG CUDA
ARG CUDNN=7.6.2.24-1
ARG CUDNN=7.6.4.38-1
ARG CUDNN_MAJOR_VERSION=7
ARG LIB_DIR_PREFIX=x86_64
ARG LIBNVINFER=6.0.1-1
ARG LIBNVINFER_MAJOR_VERSION=6
# Needed for string substitution
SHELL ["/bin/bash", "-c"]
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
cuda-command-line-tools-${CUDA/./-} \
cuda-cublas-dev-${CUDA/./-} \
libcublas10 \
libcublas-dev \
cuda-cudart-dev-${CUDA/./-} \
cuda-cufft-dev-${CUDA/./-} \
cuda-curand-dev-${CUDA/./-} \
@ -61,18 +64,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
# Install TensorRT if not building for PowerPC
RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \
apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda${CUDA} \
libnvinfer-dev=5.1.5-1+cuda${CUDA} \
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*; }
# Configure the build for our CUDA configuration.
ENV CI_BUILD_PYTHON python
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/usr/include/x64_64-linux-gnu:$LD_LIBRARY_PATH
ENV TF_NEED_CUDA 1
ENV TF_NEED_TENSORRT 1
ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
ENV TF_CUDA_VERSION=${CUDA}
ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION}
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version

20
tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile
View File

@ -22,22 +22,25 @@
ARG UBUNTU_VERSION=18.04
ARG ARCH=
ARG CUDA=10.0
ARG CUDA=10.1
FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
# ARCH and CUDA are specified again because the FROM directive resets ARGs
# (but their default value is retained if set previously)
ARG ARCH
ARG CUDA
ARG CUDNN=7.6.2.24-1
ARG CUDNN=7.6.4.38-1
ARG CUDNN_MAJOR_VERSION=7
ARG LIB_DIR_PREFIX=x86_64
ARG LIBNVINFER=6.0.1-1
ARG LIBNVINFER_MAJOR_VERSION=6
# Needed for string substitution
SHELL ["/bin/bash", "-c"]
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
cuda-command-line-tools-${CUDA/./-} \
cuda-cublas-dev-${CUDA/./-} \
libcublas10 \
libcublas-dev \
cuda-cudart-dev-${CUDA/./-} \
cuda-cufft-dev-${CUDA/./-} \
cuda-curand-dev-${CUDA/./-} \
@ -61,18 +64,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
# Install TensorRT if not building for PowerPC
RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \
apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda${CUDA} \
libnvinfer-dev=5.1.5-1+cuda${CUDA} \
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*; }
# Configure the build for our CUDA configuration.
ENV CI_BUILD_PYTHON python
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/usr/include/x64_64-linux-gnu:$LD_LIBRARY_PATH
ENV TF_NEED_CUDA 1
ENV TF_NEED_TENSORRT 1
ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
ENV TF_CUDA_VERSION=${CUDA}
ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION}
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version

2
tensorflow/tools/dockerfiles/partials/ubuntu/devel-cpu.partial.Dockerfile
View File

@ -11,7 +11,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config \
rsync \
software-properties-common \
sudo \
sudo \
unzip \
zip \
zlib1g-dev \

20
tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile
View File

@ -1,20 +1,23 @@
ARG ARCH=
ARG CUDA=10.0
ARG CUDA=10.1
FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}-base-ubuntu${UBUNTU_VERSION} as base
# ARCH and CUDA are specified again because the FROM directive resets ARGs
# (but their default value is retained if set previously)
ARG ARCH
ARG CUDA
ARG CUDNN=7.6.2.24-1
ARG CUDNN=7.6.4.38-1
ARG CUDNN_MAJOR_VERSION=7
ARG LIB_DIR_PREFIX=x86_64
ARG LIBNVINFER=6.0.1-1
ARG LIBNVINFER_MAJOR_VERSION=6
# Needed for string substitution
SHELL ["/bin/bash", "-c"]
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
cuda-command-line-tools-${CUDA/./-} \
cuda-cublas-dev-${CUDA/./-} \
libcublas10 \
libcublas-dev \
cuda-cudart-dev-${CUDA/./-} \
cuda-cufft-dev-${CUDA/./-} \
cuda-curand-dev-${CUDA/./-} \
@ -38,18 +41,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
find /usr/local/cuda-${CUDA}/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \
rm /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libcudnn_static_v7.a
# Install TensorRT if not building for PowerPC
RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \
apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda${CUDA} \
libnvinfer-dev=5.1.5-1+cuda${CUDA} \
apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin-dev=${LIBNVINFER}+cuda${CUDA} \
libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda${CUDA} \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*; }
# Configure the build for our CUDA configuration.
ENV CI_BUILD_PYTHON python
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/usr/include/x64_64-linux-gnu:$LD_LIBRARY_PATH
ENV TF_NEED_CUDA 1
ENV TF_NEED_TENSORRT 1
ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0
ENV TF_CUDA_VERSION=${CUDA}
ENV TF_CUDNN_VERSION=${CUDNN_MAJOR_VERSION}
# CACHE_STOP is used to rerun future commands, otherwise cloning tensorflow will be cached and will not pull the most recent version

35
tensorflow/tools/dockerfiles/tests/build-cpu.sh
View File

@ -15,23 +15,24 @@
# limitations under the License.
# ============================================================================
# Download and build TensorFlow.
set -euxo pipefail
git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow
set -ex
git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow || true
cd /tensorflow
ln -snf $(which ${PYTHON}) /usr/local/bin/python
# Run configure.
export TF_NEED_GCP=1
export TF_NEED_HDFS=1
export TF_NEED_S3=1
export TF_NEED_CUDA=0
# TensorRT build failing as of 2019-12-18, see
# https://github.com/tensorflow/tensorflow/issues/35115
export CC_OPT_FLAGS='-mavx'
export PYTHON_BIN_PATH=$(which python3.7)
export TMP=/tmp
yes "" | /usr/local/bin/python configure.py
ln -s $(which ${PYTHON}) /usr/local/bin/python
# For optimized builds appropriate for the hardware platform of your choosing, uncomment below...
# For ivy-bridge or sandy-bridge
# --copt=-march="ivybridge" \
# for haswell, broadwell, or skylake
# --copt=-march="haswell" \
tensorflow/tools/ci_build/builds/configured CPU \
bazel build -c opt --copt=-mavx --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
tensorflow/tools/pip_package:build_pip_package && \
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
rm -rf /tmp/pip && \
rm -rf /root/.cache
# Build the pip package and import
bazel build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" --config=opt --config=v2 tensorflow/tools/pip_package:build_pip_package
./bazel-bin/tensorflow/tools/pip_package/build_pip_package pip_pkg --gpu --nightly_flag
pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl

34
tensorflow/tools/dockerfiles/tests/build-gpu.sh
View File

@ -16,19 +16,25 @@
# ============================================================================
# Download and build TensorFlow.
set -euxo pipefail
git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow
set -ex
git clone --branch=master --depth=1 https://github.com/tensorflow/tensorflow.git /tensorflow || true
cd /tensorflow
ln -snf $(which ${PYTHON}) /usr/local/bin/python
# Run configure.
export TF_NEED_GCP=1
export TF_NEED_HDFS=1
export TF_NEED_S3=1
export TF_NEED_CUDA=1
# TensorRT build failing as of 2019-12-18, see
# https://github.com/tensorflow/tensorflow/issues/35115
export TF_NEED_TENSORRT=0
export CC_OPT_FLAGS='-mavx'
export PYTHON_BIN_PATH=$(which python3.7)
export TMP=/tmp
yes "" | /usr/local/bin/python configure.py
ln -s $(which ${PYTHON}) /usr/local/bin/python
LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \
tensorflow/tools/ci_build/builds/configured GPU \
bazel build -c opt --copt=-mavx --config=cuda \
--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
tensorflow/tools/pip_package:build_pip_package && \
rm /usr/local/cuda/lib64/stubs/libcuda.so.1 && \
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/pip && \
pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl && \
rm -rf /tmp/pip && \
rm -rf /root/.cache
# Build the pip package and import
bazel build --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" --config=opt --config=v2 tensorflow/tools/pip_package:build_pip_package
./bazel-bin/tensorflow/tools/pip_package/build_pip_package pip_pkg --gpu --nightly_flag
pip --no-cache-dir install --upgrade /tmp/pip/tensorflow-*.whl