From f5bb64336039646cae91884f7177bdcb6f29a7e2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 22 Apr 2020 01:44:56 -0700
Subject: [PATCH] Add two repository rules: - @local_execution_config_platform:
local platform to allow selecting locally executed tools on -
@local_execution_config_python: python configured for execution on the local
machine during otherwise remote builds
Mark rules that are required to run locally to require our local platform.
This allows pyth...
PiperOrigin-RevId: 307771596
Change-Id: If1f0013ec88a35d507b2b622894208aab2416fe5
---
WORKSPACE | 2 ++
tensorflow/opensource_only.files | 2 ++
tensorflow/python/BUILD | 6 +++++
tensorflow/tensorflow.bzl | 8 ++++++-
tensorflow/tools/build_info/BUILD | 1 +
tensorflow/tools/git/BUILD | 1 +
third_party/py/BUILD.tpl | 2 ++
third_party/py/python_configure.bzl | 17 ++++++++++++++
third_party/remote_config/BUILD.tpl | 17 ++++++++++++++
.../remote_platform_configure.bzl | 10 +++++++-
.../toolchains/remote_config/configs.bzl | 6 ++++-
.../toolchains/remote_config/rbe_config.bzl | 23 ++++++++++++++++++-
12 files changed, 91 insertions(+), 4 deletions(-)
diff --git a/WORKSPACE b/WORKSPACE
index 021ed6d2542..786f6fcbcf2 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -18,6 +18,8 @@ load("//tensorflow:workspace.bzl", "tf_repositories")
# Please add all new TensorFlow dependencies in workspace.bzl.
tf_repositories()
+register_execution_platforms("@local_execution_config_platform//:platform")
+register_toolchains("@local_execution_config_python//:py_toolchain")
register_toolchains("@local_config_python//:py_toolchain")
load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories")
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 3053e7c943c..d4df3df079e 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -263,6 +263,7 @@ tensorflow/third_party/toolchains/remote_config/containers.bzl
tensorflow/third_party/toolchains/remote_config/rbe_config.bzl
tensorflow/third_party/wrapt.BUILD
tensorflow/third_party/zlib.BUILD
+tensorflow/tools/build_info/BUILD
tensorflow/tools/ci_build/release/common.sh
tensorflow/tools/ci_build/release/common_win.bat
tensorflow/tools/ci_build/release/macos/cpu_libtensorflow/build.sh
@@ -393,6 +394,7 @@ tensorflow/tools/def_file_filter/BUILD
tensorflow/tools/def_file_filter/BUILD.tpl
tensorflow/tools/def_file_filter/def_file_filter.py.tpl
tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+tensorflow/tools/git/BUILD
tensorflow/tools/lib_package/BUILD
tensorflow/tools/lib_package/LibTensorFlowTest.java
tensorflow/tools/lib_package/README.md
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 8be4c143ab8..edbba09ab25 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -26,6 +26,9 @@ load("//tensorflow:tensorflow.bzl", "tf_external_workspace_visible")
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_pybind_cc_library_wrapper")
+
+# buildifier: disable=same-origin-load
+load("//tensorflow:tensorflow.bzl", "tf_local_platform_constraint")
load("//tensorflow/core/platform:build_config.bzl", "pyx_library", "tf_additional_all_protos", "tf_additional_lib_deps", "tf_proto_library", "tf_proto_library_py", "tf_protos_grappler") # @unused
load("//tensorflow/core/platform:build_config_root.bzl", "if_static", "tf_additional_plugin_deps", "tf_additional_xla_deps_py")
load("//tensorflow/python:build_defs.bzl", "tf_gen_op_wrapper_private_py")
@@ -233,9 +236,12 @@ py_library(
],
)
+# TODO(gunan): Investigate making this action hermetic so we do not need
+# to run it locally.
tf_py_build_info_genrule(
name = "py_build_info_gen",
out = "platform/build_info.py",
+ exec_compatible_with = tf_local_platform_constraint(),
)
py_library(
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 25c2b57e026..055dc10c519 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -2519,7 +2519,12 @@ def tf_genrule_cmd_append_to_srcs(to_append):
return ("cat $(SRCS) > $(@) && " + "echo >> $(@) && " + "echo " + to_append +
" >> $(@)")
+def tf_local_platform_constraint():
+ return ["@local_execution_config_platform//:platform_constraint"]
+
def tf_version_info_genrule(name, out):
+ # TODO(gunan): Investigate making this action hermetic so we do not need
+ # to run it locally.
native.genrule(
name = name,
srcs = [
@@ -2532,9 +2537,10 @@ def tf_version_info_genrule(name, out):
"$(location //tensorflow/tools/git:gen_git_source) --generate $(SRCS) \"$@\" --git_tag_override=$${GIT_TAG_OVERRIDE:-}",
local = 1,
exec_tools = [clean_dep("//tensorflow/tools/git:gen_git_source")],
+ exec_compatible_with = tf_local_platform_constraint(),
)
-def tf_py_build_info_genrule(name, out, **kwargs):
+def tf_py_build_info_genrule(name, out, exec_compatible_with, **kwargs):
native.genrule(
name = name,
outs = [out],
diff --git a/tensorflow/tools/build_info/BUILD b/tensorflow/tools/build_info/BUILD
index 91627d41d04..eb5f909a7dd 100644
--- a/tensorflow/tools/build_info/BUILD
+++ b/tensorflow/tools/build_info/BUILD
@@ -9,6 +9,7 @@ package(
py_binary(
name = "gen_build_info",
srcs = ["gen_build_info.py"],
+ exec_compatible_with = ["@local_execution_config_platform//:platform_constraint"],
python_version = "PY3",
srcs_version = "PY2AND3",
deps = [
diff --git a/tensorflow/tools/git/BUILD b/tensorflow/tools/git/BUILD
index c1f0577f33b..90eb3003f5a 100644
--- a/tensorflow/tools/git/BUILD
+++ b/tensorflow/tools/git/BUILD
@@ -11,6 +11,7 @@ package(
py_binary(
name = "gen_git_source",
srcs = ["gen_git_source.py"],
+ exec_compatible_with = ["@local_execution_config_platform//:platform_constraint"],
python_version = "PY3",
srcs_version = "PY2AND3",
)
diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl
index 3a0be1b0173..08ba167a998 100644
--- a/third_party/py/BUILD.tpl
+++ b/third_party/py/BUILD.tpl
@@ -28,6 +28,8 @@ toolchain(
name = "py_toolchain",
toolchain = ":py_runtime_pair",
toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+ target_compatible_with = [%{PLATFORM_CONSTRAINT}],
+ exec_compatible_with = [%{PLATFORM_CONSTRAINT}],
)
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl
index 6e9a22f8063..2f75262ea9f 100644
--- a/third_party/py/python_configure.bzl
+++ b/third_party/py/python_configure.bzl
@@ -240,11 +240,15 @@ def _create_local_python_repository(repository_ctx):
"numpy_include",
)
+ platform_constraint = ""
+ if repository_ctx.attr.platform_constraint:
+ platform_constraint = "\"%s\"" % repository_ctx.attr.platform_constraint
repository_ctx.template("BUILD", build_tpl, {
"%{PYTHON_BIN_PATH}": python_bin,
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
"%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule,
+ "%{PLATFORM_CONSTRAINT}": platform_constraint,
})
def _create_remote_python_repository(repository_ctx, remote_config_repo):
@@ -268,18 +272,31 @@ _ENVIRONS = [
PYTHON_LIB_PATH,
]
+local_python_configure = repository_rule(
+ implementation = _create_local_python_repository,
+ environ = _ENVIRONS,
+ attrs = {
+ "environ": attr.string_dict(),
+ "platform_constraint": attr.string(),
+ },
+)
+
remote_python_configure = repository_rule(
implementation = _create_local_python_repository,
environ = _ENVIRONS,
remotable = True,
attrs = {
"environ": attr.string_dict(),
+ "platform_constraint": attr.string(),
},
)
python_configure = repository_rule(
implementation = _python_autoconf_impl,
environ = _ENVIRONS + [TF_PYTHON_CONFIG_REPO],
+ attrs = {
+ "platform_constraint": attr.string(),
+ },
)
"""Detects and configures the local Python.
diff --git a/third_party/remote_config/BUILD.tpl b/third_party/remote_config/BUILD.tpl
index 7bcee410756..e7f93008431 100644
--- a/third_party/remote_config/BUILD.tpl
+++ b/third_party/remote_config/BUILD.tpl
@@ -1,8 +1,25 @@
+# Each platform creates a constraint @<platform>//:platform_constraint that
+# is listed in its constraint_values; rule that want to select a specific
+# platform to run on can put @<platform>//:platform_constraing into their
+# exec_compatible_with attribute.
+# Toolchains can similarly be marked with target_compatible_with or
+# exec_compatible_with to bind them to this platform.
+constraint_setting(
+ name = "platform_setting"
+)
+
+constraint_value(
+ name = "platform_constraint",
+ constraint_setting = ":platform_setting",
+ visibility = ["//visibility:public"],
+)
+
platform(
name = "platform",
constraint_values = [
"@bazel_tools//platforms:x86_64",
"@bazel_tools//platforms:%{platform}",
+ ":platform_constraint",
],
exec_properties = %{exec_properties},
)
diff --git a/third_party/remote_config/remote_platform_configure.bzl b/third_party/remote_config/remote_platform_configure.bzl
index 5c2918b8e84..1031fbfb2d9 100644
--- a/third_party/remote_config/remote_platform_configure.bzl
+++ b/third_party/remote_config/remote_platform_configure.bzl
@@ -2,6 +2,14 @@
def _remote_platform_configure_impl(repository_ctx):
platform = repository_ctx.attr.platform
+ if platform == "local":
+ os = repository_ctx.os.name.lower()
+ if os.startswith("windows"):
+ platform = "windows"
+ elif os.startswith("mac os"):
+ platform = "osx"
+ else:
+ platform = "linux"
exec_properties = repository_ctx.attr.platform_exec_properties
serialized_exec_properties = "{"
@@ -22,6 +30,6 @@ remote_platform_configure = repository_rule(
implementation = _remote_platform_configure_impl,
attrs = {
"platform_exec_properties": attr.string_dict(mandatory = True),
- "platform": attr.string(default = "linux", values = ["linux", "windows"]),
+ "platform": attr.string(default = "linux", values = ["linux", "windows", "local"]),
},
)
diff --git a/third_party/toolchains/remote_config/configs.bzl b/third_party/toolchains/remote_config/configs.bzl
index 4c94abf45b3..c97d47f64d6 100644
--- a/third_party/toolchains/remote_config/configs.bzl
+++ b/third_party/toolchains/remote_config/configs.bzl
@@ -1,8 +1,12 @@
"""Configurations of RBE builds used with remote config."""
-load("//third_party/toolchains/remote_config:rbe_config.bzl", "tensorflow_rbe_config", "tensorflow_rbe_win_config")
+load("//third_party/toolchains/remote_config:rbe_config.bzl", "tensorflow_local_config", "tensorflow_rbe_config", "tensorflow_rbe_win_config")
def initialize_rbe_configs():
+ tensorflow_local_config(
+ name = "local_execution",
+ )
+
tensorflow_rbe_config(
name = "ubuntu16.04-manylinux2010-py3",
os = "ubuntu16.04-manylinux2010",
diff --git a/third_party/toolchains/remote_config/rbe_config.bzl b/third_party/toolchains/remote_config/rbe_config.bzl
index 6709cad4eb3..597ab558eb7 100644
--- a/third_party/toolchains/remote_config/rbe_config.bzl
+++ b/third_party/toolchains/remote_config/rbe_config.bzl
@@ -1,6 +1,6 @@
"""Macro that creates external repositories for remote config."""
-load("//third_party/py:python_configure.bzl", "remote_python_configure")
+load("//third_party/py:python_configure.bzl", "local_python_configure", "remote_python_configure")
load("//third_party/gpus:cuda_configure.bzl", "remote_cuda_configure")
load("//third_party/nccl:nccl_configure.bzl", "remote_nccl_configure")
load("//third_party/gpus:rocm_configure.bzl", "remote_rocm_configure")
@@ -113,6 +113,7 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
+ platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
remote_rocm_configure(
@@ -127,10 +128,17 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No
"Pool": "default",
}
+ remote_platform_configure(
+ name = "%s_config_platform" % name,
+ platform = "linux",
+ platform_exec_properties = exec_properties,
+ )
+
remote_python_configure(
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
+ platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
else:
fail("Neither cuda_version, rocm_version nor python_version specified.")
@@ -156,7 +164,20 @@ def _tensorflow_rbe_win_config(name, python_bin_path, container_name = "windows-
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
+ platform_constraint = "@%s_config_platform//:platform_constraint" % name,
+ )
+
+def _tensorflow_local_config(name):
+ remote_platform_configure(
+ name = "%s_config_platform" % name,
+ platform = "local",
+ platform_exec_properties = {},
+ )
+ local_python_configure(
+ name = "%s_config_python" % name,
+ platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
tensorflow_rbe_config = _tensorflow_rbe_config
tensorflow_rbe_win_config = _tensorflow_rbe_win_config
+tensorflow_local_config = _tensorflow_local_config