Merge pull request #43586 from Intel-tensorflow:agramesh/remove_blob_linux

PiperOrigin-RevId: 337575441
Change-Id: If35ef802fce3e7d8cc2403593db31bd6b0465235

tensorflow/opensource_only.files

@@ -117,6 +117,7 @@ tensorflow/third_party/llvm/BUILD
 tensorflow/third_party/llvm/expand_cmake_vars.py
 tensorflow/third_party/llvm/llvm.autogenerated.BUILD
 tensorflow/third_party/llvm/llvm.bzl
+tensorflow/third_party/llvm_openmp/BUILD
 tensorflow/third_party/lmdb.BUILD
 tensorflow/third_party/mkl/BUILD
 tensorflow/third_party/mkl/LICENSE

tensorflow/tensorflow.bzl

@@ -330,8 +330,7 @@ def tf_copts(
         if_libtpu(["-DLIBTPU_ON_GCE"], []) +
         if_xla_available(["-DTENSORFLOW_USE_XLA=1"]) +
         if_tensorrt(["-DGOOGLE_TENSORRT=1"]) +
-        if_mkl(["-DINTEL_MKL=1", "-DENABLE_MKLDNN_V1", "-DENABLE_INTEL_MKL_BFLOAT16"]) +
-        if_mkl_open_source_only(["-DINTEL_MKL_DNN_ONLY"]) +
+        if_mkl(["-DINTEL_MKL=1", "-DENABLE_MKLDNN_V1", "-DENABLE_INTEL_MKL_BFLOAT16", "-DINTEL_MKL_DNN_ONLY"]) +
         if_mkldnn_threadpool(["-DENABLE_MKLDNN_THREADPOOL"]) +
         if_enable_mkl(["-DENABLE_MKL"]) +
         if_ngraph(["-DINTEL_NGRAPH=1"]) +

tensorflow/workspace.bzl

@@ -125,16 +125,6 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
         armhf_repo = "../armhf_linux_toolchain",
     )
 
-    mkl_repository(
-        name = "mkl_linux",
-        build_file = clean_dep("//third_party/mkl:mkl.BUILD"),
-        sha256 = "a936d6b277a33d2a027a024ea8e65df62bd2e162c7ca52c48486ed9d5dc27160",
-        strip_prefix = "mklml_lnx_2019.0.5.20190502",
-        urls = [
-            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/intel/mkl-dnn/releases/download/v0.21/mklml_lnx_2019.0.5.20190502.tgz",
-            "https://github.com/intel/mkl-dnn/releases/download/v0.21/mklml_lnx_2019.0.5.20190502.tgz",
-        ],
-    )
     mkl_repository(
         name = "mkl_windows",
         build_file = clean_dep("//third_party/mkl:mkl.BUILD"),
@@ -730,6 +720,18 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
         },
     )
 
+    # Intel openMP that is part of LLVM sources.
+    tf_http_archive(
+        name = "llvm_openmp",
+        build_file = clean_dep("//third_party/llvm_openmp:BUILD"),
+        sha256 = "d19f728c8e04fb1e94566c8d76aef50ec926cd2f95ef3bf1e0a5de4909b28b44",
+        strip_prefix = "openmp-10.0.1.src",
+        urls = [
+            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/releases/download/llvmorg-10.0.1/openmp-10.0.1.src.tar.xz",
+            "https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.1/openmp-10.0.1.src.tar.xz",
+        ],
+    )
+
     tf_http_archive(
         name = "lmdb",
         build_file = clean_dep("//third_party:lmdb.BUILD"),

third_party/llvm/BUILD

@@ -2,5 +2,8 @@ py_binary(
     name = "expand_cmake_vars",
     srcs = ["expand_cmake_vars.py"],
     srcs_version = "PY2AND3",
-    visibility = ["@llvm-project//:__subpackages__"],
+    visibility = [
+        "@llvm-project//:__subpackages__",
+        "@llvm_openmp//:__subpackages__",
+    ],
 )

third_party/llvm/expand_cmake_vars.py

@@ -25,6 +25,7 @@ import sys
 _CMAKE_DEFINE_REGEX = re.compile(r"\s*#cmakedefine\s+([A-Za-z_0-9]*)(\s.*)?$")
 _CMAKE_DEFINE01_REGEX = re.compile(r"\s*#cmakedefine01\s+([A-Za-z_0-9]*)")
 _CMAKE_VAR_REGEX = re.compile(r"\${([A-Za-z_0-9]*)}")
+_CMAKE_ATVAR_REGEX = re.compile(r"@([A-Za-z_0-9]*)@")
 
 
 def _parse_args(argv):
@@ -37,10 +38,10 @@ def _parse_args(argv):
 
 
 def _expand_variables(input_str, cmake_vars):
-  """Expands ${VARIABLE}s in 'input_str', using dictionary 'cmake_vars'.
+  """Expands ${VARIABLE}s and @VARIABLE@s in 'input_str', using dictionary 'cmake_vars'.
 
   Args:
-    input_str: the string containing ${VARIABLE} expressions to expand.
+    input_str: the string containing ${VARIABLE} or @VARIABLE@ expressions to expand.
     cmake_vars: a dictionary mapping variable names to their values.
 
   Returns:
@@ -50,7 +51,7 @@ def _expand_variables(input_str, cmake_vars):
     if match.group(1) in cmake_vars:
       return cmake_vars[match.group(1)]
     return ""
-  return _CMAKE_VAR_REGEX.sub(replace, input_str)
+  return _CMAKE_ATVAR_REGEX.sub(replace,_CMAKE_VAR_REGEX.sub(replace, input_str))
 
 
 def _expand_cmakedefines(line, cmake_vars):

third_party/llvm_openmp/BUILD

@@ -0,0 +1,127 @@
+# Build file for OpenMP library that is part of llvm
+load(
+    "@org_tensorflow//third_party/llvm:llvm.bzl",
+    "cmake_var_string",
+    "expand_cmake_vars",
+)
+
+exports_files(["LICENSE.txt"])
+
+genrule(
+    name = "kmp_i18n_id",
+    srcs = [
+        "runtime/tools/message-converter.pl",
+        "runtime/src/i18n/en_US.txt",
+    ],
+    outs = ["include/kmp_i18n_id.inc"],
+    cmd = "perl $(location runtime/tools/message-converter.pl) --os=lin --prefix=kmp_i18n --enum=$@  $(location runtime/src/i18n/en_US.txt)",
+)
+
+genrule(
+    name = "kmp_i18n_default",
+    srcs = [
+        "runtime/tools/message-converter.pl",
+        "runtime/src/i18n/en_US.txt",
+    ],
+    outs = ["include/kmp_i18n_default.inc"],
+    cmd = "perl $(location runtime/tools/message-converter.pl) --os=lin --prefix=kmp_i18n --default=$@ $(location runtime/src/i18n/en_US.txt)",
+)
+
+# Bazel doesn't accept .txt as an input, rename the ldscript to .inc to workaround.
+genrule(
+    name = "ldscript",
+    srcs = ["runtime/src/exports_so.txt"],
+    outs = ["exports_so.inc"],
+    cmd = "cp $(location runtime/src/exports_so.txt) $@",
+)
+
+# Cmake vars to replace.
+omp_vars = {
+    "LIBOMP_USE_VERSION_SYMBOLS": 1,
+    "LIBOMP_HAVE_WEAK_ATTRIBUTE": 1,
+    "LIBOMP_USE_ADAPTIVE_LOCKS": 1,
+    "LIBOMP_ENABLE_ASSERTIONS": 1,
+    "LIBOMP_ENABLE_SHARED": 1,
+    "LIBOMP_LEGAL_ARCH": "Intel(R) 64",
+    "LIBOMP_LIB_FILE": "libiomp5",
+    "LIBOMP_VERSION_MAJOR": 5,
+    "LIBOMP_VERSION_MINOR": 0,
+}
+
+omp_all_cmake_vars = cmake_var_string(omp_vars)
+
+expand_cmake_vars(
+    name = "config_kmp",
+    src = "runtime/src/kmp_config.h.cmake",
+    cmake_vars = omp_all_cmake_vars,
+    dst = "include/kmp_config.h",
+)
+
+expand_cmake_vars(
+    name = "config_omp",
+    src = "runtime/src/include/omp.h.var",
+    cmake_vars = omp_all_cmake_vars,
+    dst = "include/omp.h",
+)
+
+# TODO(Intel-tf) Replace the following cc_binary call with cc_library.
+# cc_library should be used for files that are not independently executed. Using
+# cc_library here results in the following linking errors.
+# ERROR: //tensorflow/BUILD:689:1: Linking of rule '//tensorflow:libtensorflow_framework.so.2.4.0' failed (Exit 1)
+# /usr/bin/ld.gold: error: symbol GOMP_parallel_loop_nonmonotonic_guided has undefined version VERSION
+# /usr/bin/ld.gold: error: symbol GOMP_parallel_start has undefined version GOMP_1.0
+# /usr/bin/ld.gold: error: symbol GOMP_cancellation_point has undefined version GOMP_4.0
+# /usr/bin/ld.gold: error: symbol omp_set_num_threads has undefined version OMP_1.0
+# ......
+# ......
+
+cc_binary(
+    name = "libiomp5.so",
+    srcs = [
+        ":config_kmp",
+        ":config_omp",
+        ":kmp_i18n_id",
+        ":kmp_i18n_default",
+        ":ldscript",
+        "runtime/src/kmp_alloc.cpp",
+        "runtime/src/kmp_atomic.cpp",
+        "runtime/src/kmp_csupport.cpp",
+        "runtime/src/kmp_debug.cpp",
+        "runtime/src/kmp_itt.cpp",
+        "runtime/src/kmp_environment.cpp",
+        "runtime/src/kmp_error.cpp",
+        "runtime/src/kmp_global.cpp",
+        "runtime/src/kmp_i18n.cpp",
+        "runtime/src/kmp_io.cpp",
+        "runtime/src/kmp_runtime.cpp",
+        "runtime/src/kmp_settings.cpp",
+        "runtime/src/kmp_str.cpp",
+        "runtime/src/kmp_tasking.cpp",
+        "runtime/src/kmp_threadprivate.cpp",
+        "runtime/src/kmp_utility.cpp",
+        "runtime/src/kmp_barrier.cpp",
+        "runtime/src/kmp_wait_release.cpp",
+        "runtime/src/kmp_affinity.cpp",
+        "runtime/src/kmp_dispatch.cpp",
+        "runtime/src/kmp_lock.cpp",
+        "runtime/src/kmp_sched.cpp",
+        "runtime/src/kmp_taskdeps.cpp",
+        "runtime/src/kmp_cancel.cpp",
+        "runtime/src/kmp_ftn_cdecl.cpp",
+        "runtime/src/kmp_ftn_extra.cpp",
+        "runtime/src/kmp_version.cpp",
+
+        #linux specific files
+        "runtime/src/z_Linux_util.cpp",
+        "runtime/src/kmp_gsupport.cpp",
+        "runtime/src/z_Linux_asm.S",
+    ],
+    copts = ["-Domp_EXPORTS -D_GNU_SOURCE -D_REENTRANT"],
+    includes = [
+        "include/",
+        "runtime/src/",
+    ],
+    linkopts = ["-lpthread -ldl -Wl,--version-script=$(location :ldscript)"],
+    linkshared = True,
+    visibility = ["//visibility:public"],
+)

third_party/mkl/BUILD

@@ -42,7 +42,7 @@ filegroup(
     name = "LICENSE",
     srcs = ["MKL_LICENSE"] + select({
         "@org_tensorflow//tensorflow:linux_x86_64": [
-            "@mkl_linux//:LICENSE",
+            "@llvm_openmp//:LICENSE.txt",
         ],
         "@org_tensorflow//tensorflow:macos": [
             "@mkl_darwin//:LICENSE",
@@ -55,13 +55,23 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 
+# TODO(Intel-tf) Remove the following call to cc_library and replace all uses
+# of mkl_libs_linux with @llvm_openmp//:libiomp5.so directly.
+
+cc_library(
+    name = "mkl_libs_linux",
+    srcs = [
+        "@llvm_openmp//:libiomp5.so",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 cc_library(
     name = "intel_binary_blob",
     visibility = ["//visibility:public"],
     deps = select({
         "@org_tensorflow//tensorflow:linux_x86_64": [
-            "@mkl_linux//:mkl_headers",
-            "@mkl_linux//:mkl_libs_linux",
+            ":mkl_libs_linux",
         ],
         "@org_tensorflow//tensorflow:macos": [
             "@mkl_darwin//:mkl_headers",

third_party/mkl/mkl.BUILD

@@ -17,15 +17,6 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-cc_library(
-    name = "mkl_libs_linux",
-    srcs = [
-        "lib/libiomp5.so",
-        "lib/libmklml_intel.so",
-    ],
-    visibility = ["//visibility:public"],
-)
-
 cc_library(
     name = "mkl_libs_darwin",
     srcs = [