Merge pull request #45800 from ashahba:ashahba/fix-horovod

PiperOrigin-RevId: 349669397 Change-Id: Id4c9fc84d774e1db26f85196dee018251ff9819a
2020-12-31 05:15:22 -08:00 · 2020-12-31 05:15:22 -08:00 · 82c642bcbb
commit 82c642bcbb
parent eddde5d462 c21a9d30f7
23 changed files with 34 additions and 9 deletions
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-jupyter.Dockerfile
@ -27,7 +27,7 @@ ARG CENTOS_VERSION=8

 # Enable both PowerTools and EPEL otherwise some packages like hdf5-devel fail to install
 RUN dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --set-enabled PowerTools && \
+    dnf config-manager --set-enabled powertools && \
    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-"${CENTOS_VERSION}".noarch.rpm && \
    dnf clean all

@ -40,6 +40,7 @@ RUN yum update -y && \
        git \
        hdf5-devel \
        java-1.8.0-openjdk \
+        java-1.8.0-openjdk-devel \
        java-1.8.0-openjdk-headless \
        libcurl-devel \
        make \
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-mpi-horovod-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-mpi-horovod-jupyter.Dockerfile
@ -27,7 +27,7 @@ ARG CENTOS_VERSION=8

 # Enable both PowerTools and EPEL otherwise some packages like hdf5-devel fail to install
 RUN dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --set-enabled PowerTools && \
+    dnf config-manager --set-enabled powertools && \
    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-"${CENTOS_VERSION}".noarch.rpm && \
    dnf clean all

@ -40,6 +40,7 @@ RUN yum update -y && \
        git \
        hdf5-devel \
        java-1.8.0-openjdk \
+        java-1.8.0-openjdk-devel \
        java-1.8.0-openjdk-headless \
        libcurl-devel \
        make \
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-mpi-horovod.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-mpi-horovod.Dockerfile
@ -27,7 +27,7 @@ ARG CENTOS_VERSION=8

 # Enable both PowerTools and EPEL otherwise some packages like hdf5-devel fail to install
 RUN dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --set-enabled PowerTools && \
+    dnf config-manager --set-enabled powertools && \
    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-"${CENTOS_VERSION}".noarch.rpm && \
    dnf clean all

@ -40,6 +40,7 @@ RUN yum update -y && \
        git \
        hdf5-devel \
        java-1.8.0-openjdk \
+        java-1.8.0-openjdk-devel \
        java-1.8.0-openjdk-headless \
        libcurl-devel \
        make \
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-mpich-horovod-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-mpich-horovod-jupyter.Dockerfile
@ -27,7 +27,7 @@ ARG CENTOS_VERSION=8

 # Enable both PowerTools and EPEL otherwise some packages like hdf5-devel fail to install
 RUN dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --set-enabled PowerTools && \
+    dnf config-manager --set-enabled powertools && \
    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-"${CENTOS_VERSION}".noarch.rpm && \
    dnf clean all

@ -40,6 +40,7 @@ RUN yum update -y && \
        git \
        hdf5-devel \
        java-1.8.0-openjdk \
+        java-1.8.0-openjdk-devel \
        java-1.8.0-openjdk-headless \
        libcurl-devel \
        make \
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-mpich-horovod.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel-mpich-horovod.Dockerfile
@ -27,7 +27,7 @@ ARG CENTOS_VERSION=8

 # Enable both PowerTools and EPEL otherwise some packages like hdf5-devel fail to install
 RUN dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --set-enabled PowerTools && \
+    dnf config-manager --set-enabled powertools && \
    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-"${CENTOS_VERSION}".noarch.rpm && \
    dnf clean all

@ -40,6 +40,7 @@ RUN yum update -y && \
        git \
        hdf5-devel \
        java-1.8.0-openjdk \
+        java-1.8.0-openjdk-devel \
        java-1.8.0-openjdk-headless \
        libcurl-devel \
        make \
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-devel.Dockerfile
@ -27,7 +27,7 @@ ARG CENTOS_VERSION=8

 # Enable both PowerTools and EPEL otherwise some packages like hdf5-devel fail to install
 RUN dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --set-enabled PowerTools && \
+    dnf config-manager --set-enabled powertools && \
    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-"${CENTOS_VERSION}".noarch.rpm && \
    dnf clean all

@ -40,6 +40,7 @@ RUN yum update -y && \
        git \
        hdf5-devel \
        java-1.8.0-openjdk \
+        java-1.8.0-openjdk-devel \
        java-1.8.0-openjdk-headless \
        libcurl-devel \
        make \
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-mpi-horovod-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-mpi-horovod-jupyter.Dockerfile
@ -88,8 +88,10 @@ ARG HOROVOD_WITH_TENSORFLOW=1
 ARG HOROVOD_VERSION=

 RUN yum update -y && yum install -y \
+    cmake \
    gcc \
    gcc-c++ \
+    make \
    python36-devel && \
    yum clean all

--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-mpi-horovod.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-mpi-horovod.Dockerfile
@ -88,8 +88,10 @@ ARG HOROVOD_WITH_TENSORFLOW=1
 ARG HOROVOD_VERSION=

 RUN yum update -y && yum install -y \
+    cmake \
    gcc \
    gcc-c++ \
+    make \
    python36-devel && \
    yum clean all

--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-mpich-horovod-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-mpich-horovod-jupyter.Dockerfile
@ -87,8 +87,10 @@ ARG HOROVOD_WITH_TENSORFLOW=1
 ARG HOROVOD_VERSION=

 RUN yum update -y && yum install -y \
+    cmake \
    gcc \
    gcc-c++ \
+    make \
    python36-devel && \
    yum clean all

--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-mpich-horovod.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/centos-8-mpich-horovod.Dockerfile
@ -87,8 +87,10 @@ ARG HOROVOD_WITH_TENSORFLOW=1
 ARG HOROVOD_VERSION=

 RUN yum update -y && yum install -y \
+    cmake \
    gcc \
    gcc-c++ \
+    make \
    python36-devel && \
    yum clean all

--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-18.04-mpi-horovod-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-18.04-mpi-horovod-jupyter.Dockerfile
@ -85,6 +85,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    python3-dev
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-18.04-mpi-horovod.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-18.04-mpi-horovod.Dockerfile
@ -85,6 +85,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    python3-dev
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-18.04-mpich-horovod-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-18.04-mpich-horovod-jupyter.Dockerfile
@ -81,6 +81,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    python3-dev
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-18.04-mpich-horovod.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-18.04-mpich-horovod.Dockerfile
@ -81,6 +81,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    python3-dev
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-20.04-mpi-horovod-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-20.04-mpi-horovod-jupyter.Dockerfile
@ -95,6 +95,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    ${PYTHON}-dev
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-20.04-mpi-horovod.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-20.04-mpi-horovod.Dockerfile
@ -95,6 +95,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    ${PYTHON}-dev
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-20.04-mpich-horovod-jupyter.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-20.04-mpich-horovod-jupyter.Dockerfile
@ -91,6 +91,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    ${PYTHON}-dev
--- a/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-20.04-mpich-horovod.Dockerfile
+++ b/tensorflow/tools/dockerfiles/dockerfiles/onednn/ubuntu-20.04-mpich-horovod.Dockerfile
@ -91,6 +91,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    ${PYTHON}-dev
--- a/tensorflow/tools/dockerfiles/partials/onednn/centos/devel.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/onednn/centos/devel.partial.Dockerfile
@ -4,7 +4,7 @@ ARG CENTOS_VERSION=8

 # Enable both PowerTools and EPEL otherwise some packages like hdf5-devel fail to install
 RUN dnf install -y 'dnf-command(config-manager)' && \
-    dnf config-manager --set-enabled PowerTools && \
+    dnf config-manager --set-enabled powertools && \
    dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-"${CENTOS_VERSION}".noarch.rpm && \
    dnf clean all

@ -17,6 +17,7 @@ RUN yum update -y && \
        git \
        hdf5-devel \
        java-1.8.0-openjdk \
+        java-1.8.0-openjdk-devel \
        java-1.8.0-openjdk-headless \
        libcurl-devel \
        make \
--- a/tensorflow/tools/dockerfiles/partials/onednn/centos/horovod.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/onednn/centos/horovod.partial.Dockerfile
@ -5,8 +5,10 @@ ARG HOROVOD_WITH_TENSORFLOW=1
 ARG HOROVOD_VERSION=

 RUN yum update -y && yum install -y \
+    cmake \
    gcc \
    gcc-c++ \
+    make \
    python36-devel && \
    yum clean all

--- a/tensorflow/tools/dockerfiles/partials/onednn/ubuntu/2004-horovod.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/onednn/ubuntu/2004-horovod.partial.Dockerfile
@ -6,6 +6,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    ${PYTHON}-dev
--- a/tensorflow/tools/dockerfiles/partials/onednn/ubuntu/horovod.partial.Dockerfile
+++ b/tensorflow/tools/dockerfiles/partials/onednn/ubuntu/horovod.partial.Dockerfile
@ -6,6 +6,7 @@ ARG HOROVOD_VERSION=

 RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
    build-essential \
+    cmake \
    g++-8 \
    gcc-8 \
    python3-dev
--- a/tensorflow/tools/dockerfiles/spec.yml
+++ b/tensorflow/tools/dockerfiles/spec.yml
@ -351,7 +351,6 @@ slice_sets:
          args:
              - CENTOS_VERSION=8
              - PYTHON=python3
-              - HOROVOD_VERSION=0.19.5
              - TF_PACKAGE=intel-tensorflow

    devel-onednn-mpi-horovod:
@ -499,7 +498,6 @@ slice_sets:
          args:
              - CENTOS_VERSION=8
              - PYTHON=python3
-              - HOROVOD_VERSION=0.19.5
              - TF_PACKAGE=intel-tensorflow

    devel-onednn-mpich-horovod: