From 9e651e4571f7b7c2d32bdafe43cc4ced9bb0c750 Mon Sep 17 00:00:00 2001
From: Ilya Biryukov <ibiryukov@google.com>
Date: Thu, 22 Mar 2018 05:33:42 -0700
Subject: [PATCH] Allow to download clang and use clang for CPU builds.
Previously we only allowed to download clang when doing GPU builds.
The added skylark files use bazel's autoconf scripts, which were only added in
0.10.0. To provide nice error message for older versions of bazel
(i.e. 'version is less than 0.10' vs 'can't load @bazel_tools/cpp/...'), we
move the bazel version check into WORKSPACE file from workspace.bzl.
PiperOrigin-RevId: 190050798
---
WORKSPACE | 6 +++
configure.py | 26 ++++-----
tensorflow/version_check.bzl | 48 +++++++++++++++++
tensorflow/workspace.bzl | 53 ++-----------------
third_party/clang_toolchain/BUILD | 0
.../clang_toolchain/cc_configure_clang.bzl | 27 ++++++++++
.../download_clang.bzl | 0
third_party/gpus/cuda_configure.bzl | 2 +-
third_party/mkl_dnn/mkldnn.BUILD | 2 +-
tools/bazel.rc | 5 +-
10 files changed, 104 insertions(+), 65 deletions(-)
create mode 100644 tensorflow/version_check.bzl
create mode 100644 third_party/clang_toolchain/BUILD
create mode 100644 third_party/clang_toolchain/cc_configure_clang.bzl
rename third_party/{gpus => clang_toolchain}/download_clang.bzl (100%)
diff --git a/WORKSPACE b/WORKSPACE
index 1e38a9a8cd7..11c5cdb2070 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -14,6 +14,12 @@ load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories")
closure_repositories()
+# We must check the bazel version before trying to parse any other BUILD
+# files, in case the parsing of those build files depends on the bazel
+# version we require here.
+load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
+check_bazel_version_at_least("0.10.0")
+
load("//tensorflow:workspace.bzl", "tf_workspace")
# Uncomment and update the paths in these entries to build the Android demo.
diff --git a/configure.py b/configure.py
index 7d61c2e5e3a..ea732c64e2a 100644
--- a/configure.py
+++ b/configure.py
@@ -524,7 +524,7 @@ def set_tf_cuda_clang(environ_cp):
def set_tf_download_clang(environ_cp):
"""Set TF_DOWNLOAD_CLANG action_env."""
- question = 'Do you want to download a fresh release of clang? (Experimental)'
+ question = 'Do you wish to download a fresh release of clang? (Experimental)'
yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
no_reply = 'Clang will not be downloaded.'
set_action_env_var(
@@ -1380,7 +1380,7 @@ def main():
# environment variables.
environ_cp = dict(os.environ)
- check_bazel_version('0.5.4')
+ check_bazel_version('0.10.0')
reset_tf_configure_bazelrc(args.workspace)
cleanup_makefile()
@@ -1397,6 +1397,9 @@ def main():
environ_cp['TF_NEED_OPENCL'] = '0'
environ_cp['TF_CUDA_CLANG'] = '0'
environ_cp['TF_NEED_TENSORRT'] = '0'
+ # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
+ # Windows.
+ environ_cp['TF_DOWNLOAD_CLANG'] = '0'
if is_macos():
environ_cp['TF_NEED_JEMALLOC'] = '0'
@@ -1444,16 +1447,8 @@ def main():
set_tf_cuda_clang(environ_cp)
if environ_cp.get('TF_CUDA_CLANG') == '1':
- if not is_windows():
- # Ask if we want to download clang release while building.
- set_tf_download_clang(environ_cp)
- else:
- # We use bazel's generated crosstool on Windows and there is no
- # way to provide downloaded toolchain for that yet.
- # TODO(ibiryukov): Investigate using clang as a cuda compiler on
- # Windows.
- environ_cp['TF_DOWNLOAD_CLANG'] = '0'
-
+ # Ask whether we should download the clang toolchain.
+ set_tf_download_clang(environ_cp)
if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
# Set up which clang we should use as the cuda / host compiler.
set_clang_cuda_compiler_path(environ_cp)
@@ -1463,6 +1458,13 @@ def main():
if not is_windows():
set_gcc_host_compiler_path(environ_cp)
set_other_cuda_vars(environ_cp)
+ else:
+ # CUDA not required. Ask whether we should download the clang toolchain and
+ # use it for the CPU build.
+ set_tf_download_clang(environ_cp)
+ if environ_cp.get('TF_DOWNLOAD_CLANG') == '1':
+ write_to_bazelrc('build --config=download_clang')
+ write_to_bazelrc('test --config=download_clang')
set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False)
if environ_cp.get('TF_NEED_MPI') == '1':
diff --git a/tensorflow/version_check.bzl b/tensorflow/version_check.bzl
new file mode 100644
index 00000000000..79e721dab42
--- /dev/null
+++ b/tensorflow/version_check.bzl
@@ -0,0 +1,48 @@
+""" Helpers to check minimum version of bazel."""
+
+def _extract_version_number(bazel_version):
+ """Extracts the semantic version number from a version string
+
+ Args:
+ bazel_version: the version string that begins with the semantic version
+ e.g. "1.2.3rc1 abc1234" where "abc1234" is a commit hash.
+
+ Returns:
+ The semantic version string, like "1.2.3".
+ """
+ for i in range(len(bazel_version)):
+ c = bazel_version[i]
+ if not (c.isdigit() or c == "."):
+ return bazel_version[:i]
+ return bazel_version
+
+# Parse the bazel version string from `native.bazel_version`.
+# e.g.
+# "0.10.0rc1 abc123d" => (0, 10, 0)
+# "0.3.0" => (0, 3, 0)
+def _parse_bazel_version(bazel_version):
+ """Parses a version string into a 3-tuple of ints
+
+ int tuples can be compared directly using binary operators (<, >).
+
+ Args:
+ bazel_version: the Bazel version string
+
+ Returns:
+ An int 3-tuple of a (major, minor, patch) version.
+ """
+
+ version = _extract_version_number(bazel_version)
+ return tuple([int(n) for n in version.split(".")])
+
+def check_bazel_version_at_least(minimum_bazel_version):
+ if "bazel_version" not in dir(native):
+ fail("\nCurrent Bazel version is lower than 0.2.1, expected at least %s\n" % minimum_bazel_version)
+ elif not native.bazel_version:
+ print("\nCurrent Bazel is not a release version, cannot check for compatibility.")
+ print("Make sure that you are running at least Bazel %s.\n" % minimum_bazel_version)
+ return
+
+ if _parse_bazel_version(native.bazel_version) < _parse_bazel_version(minimum_bazel_version):
+ fail("\nCurrent Bazel version is {}, expected at least {}\n".format(
+ native.bazel_version, minimum_bazel_version))
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 675acbe5f60..ebb9e9412f6 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -10,65 +10,18 @@ load("//third_party/sycl:sycl_configure.bzl", "sycl_configure")
load("//third_party/toolchains/clang6:repo.bzl", "clang6_configure")
load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compiler_configure")
load("//third_party:repo.bzl", "tf_http_archive")
+load("//third_party/clang_toolchain:cc_configure_clang.bzl", "cc_download_clang_toolchain")
load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external")
load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
-def _extract_version_number(bazel_version):
- """Extracts the semantic version number from a version string
-
- Args:
- bazel_version: the version string that begins with the semantic version
- e.g. "1.2.3rc1 abc1234" where "abc1234" is a commit hash.
-
- Returns:
- The semantic version string, like "1.2.3".
- """
- for i in range(len(bazel_version)):
- c = bazel_version[i]
- if not (c.isdigit() or c == "."):
- return bazel_version[:i]
- return bazel_version
-
-# Parse the bazel version string from `native.bazel_version`.
-# e.g.
-# "0.10.0rc1 abc123d" => (0, 10, 0)
-# "0.3.0" => (0, 3, 0)
-def _parse_bazel_version(bazel_version):
- """Parses a version string into a 3-tuple of ints
-
- int tuples can be compared directly using binary operators (<, >).
-
- Args:
- bazel_version: the Bazel version string
-
- Returns:
- An int 3-tuple of a (major, minor, patch) version.
- """
-
- version = _extract_version_number(bazel_version)
- return tuple([int(n) for n in version.split(".")])
-
-def check_bazel_version_at_least(minimum_bazel_version):
- if "bazel_version" not in dir(native):
- fail("\nCurrent Bazel version is lower than 0.2.1, expected at least %s\n" % minimum_bazel_version)
- elif not native.bazel_version:
- print("\nCurrent Bazel is not a release version, cannot check for compatibility.")
- print("Make sure that you are running at least Bazel %s.\n" % minimum_bazel_version)
- return
-
- if _parse_bazel_version(native.bazel_version) < _parse_bazel_version(minimum_bazel_version):
- fail("\nCurrent Bazel version is {}, expected at least {}\n".format(
- native.bazel_version, minimum_bazel_version))
# If TensorFlow is linked as a submodule.
# path_prefix is no longer used.
# tf_repo_name is thought to be under consideration.
def tf_workspace(path_prefix="", tf_repo_name=""):
- # We must check the bazel version before trying to parse any other BUILD
- # files, in case the parsing of those build files depends on the bazel
- # version we require here.
- check_bazel_version_at_least("0.5.4")
+ # Note that we check the minimum bazel version in WORKSPACE.
clang6_configure(name="local_config_clang6")
+ cc_download_clang_toolchain(name="local_config_download_clang")
cuda_configure(name="local_config_cuda")
tensorrt_configure(name="local_config_tensorrt")
git_configure(name="local_config_git")
diff --git a/third_party/clang_toolchain/BUILD b/third_party/clang_toolchain/BUILD
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/third_party/clang_toolchain/cc_configure_clang.bzl b/third_party/clang_toolchain/cc_configure_clang.bzl
new file mode 100644
index 00000000000..1181110ea96
--- /dev/null
+++ b/third_party/clang_toolchain/cc_configure_clang.bzl
@@ -0,0 +1,27 @@
+""" Downloads clang and configures the crosstool using bazel's autoconf."""
+
+load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl")
+load(":download_clang.bzl", "download_clang")
+
+_TF_DOWNLOAD_CLANG = "TF_DOWNLOAD_CLANG"
+_TF_NEED_CUDA = "TF_NEED_CUDA"
+
+def _cc_clang_autoconf(repo_ctx):
+ if repo_ctx.os.environ.get(_TF_DOWNLOAD_CLANG) != "1":
+ return
+ if repo_ctx.os.environ.get(_TF_NEED_CUDA) == "1":
+ # Clang is handled separately for CUDA configs.
+ # See cuda_configure.bzl for more details.
+ return
+
+ download_clang(repo_ctx, out_folder='extra_tools')
+ overriden_tools = {'gcc': 'extra_tools/bin/clang'}
+ cc_autoconf_impl(repo_ctx, overriden_tools)
+
+cc_download_clang_toolchain = repository_rule(
+ environ = [
+ _TF_DOWNLOAD_CLANG,
+ _TF_NEED_CUDA,
+ ],
+ implementation = _cc_clang_autoconf,
+)
diff --git a/third_party/gpus/download_clang.bzl b/third_party/clang_toolchain/download_clang.bzl
similarity index 100%
rename from third_party/gpus/download_clang.bzl
rename to third_party/clang_toolchain/download_clang.bzl
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 6c9c128db63..ede7e318976 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -96,7 +96,7 @@ NVVM_LIBDEVICE_PATHS = [
"share/cuda/",
]
-load(":download_clang.bzl", "download_clang")
+load("//third_party/clang_toolchain:download_clang.bzl", "download_clang")
# TODO(dzc): Once these functions have been factored out of Bazel's
# cc_configure.bzl, load them from @bazel_tools instead.
diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD
index 752a0d84980..68f24aabaee 100644
--- a/third_party/mkl_dnn/mkldnn.BUILD
+++ b/third_party/mkl_dnn/mkldnn.BUILD
@@ -4,7 +4,7 @@ config_setting(
name = "clang_linux_x86_64",
values = {
"cpu": "k8",
- "define": "using_cuda_clang=true",
+ "define": "using_clang=true",
},
)
diff --git a/tools/bazel.rc b/tools/bazel.rc
index 8b8c7175617..1c1e6afb65a 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -27,11 +27,14 @@ build --define framework_shared_object=true
build:mkl --define=using_mkl=true
build:mkl -c opt
+build:download_clang --crosstool_top=@local_config_download_clang//:toolchain
+build:download_clang --define=using_clang=true
+
build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain
build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true
build:cuda_clang --crosstool_top=@local_config_cuda//crosstool:toolchain
-build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true
+build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true --define=using_clang=true
build:win-cuda --define=using_cuda=true --define=using_cuda_nvcc=true