From 0f631f8fb121d7415a4f1b162078d6d6abf15763 Mon Sep 17 00:00:00 2001
From: Taehee Jeong <taeheej@google.com>
Date: Tue, 21 Apr 2020 05:17:27 -0700
Subject: [PATCH] Add two repository rules: - @local_execution_config_platform:
local platform to allow selecting locally executed tools on -
@local_execution_config_python: python configured for execution on the local
machine during otherwise remote builds
Mark rules that are required to run locally to require our local platform.
This allows python paths to differ between the remote docker image and local
machine.
For example, the local machine might have python 3.7 installed in
/usr/bin/python3, while the remote docker should use a python installed
in /usr/local/bin/python3.8.
PiperOrigin-RevId: 307585019
Change-Id: I29313121beb967b77ae123e7d1b614c688cb40ca
---
WORKSPACE | 2 --
tensorflow/opensource_only.files | 2 --
tensorflow/python/BUILD | 6 -----
tensorflow/tensorflow.bzl | 8 +------
tensorflow/tools/build_info/BUILD | 1 -
tensorflow/tools/git/BUILD | 1 -
third_party/py/BUILD.tpl | 2 --
third_party/py/python_configure.bzl | 17 --------------
third_party/remote_config/BUILD.tpl | 17 --------------
.../toolchains/remote_config/configs.bzl | 6 +----
.../toolchains/remote_config/rbe_config.bzl | 22 +------------------
11 files changed, 3 insertions(+), 81 deletions(-)
diff --git a/WORKSPACE b/WORKSPACE
index 786f6fcbcf2..021ed6d2542 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -18,8 +18,6 @@ load("//tensorflow:workspace.bzl", "tf_repositories")
# Please add all new TensorFlow dependencies in workspace.bzl.
tf_repositories()
-register_execution_platforms("@local_execution_config_platform//:platform")
-register_toolchains("@local_execution_config_python//:py_toolchain")
register_toolchains("@local_config_python//:py_toolchain")
load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories")
diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files
index 0b287c9824a..1e45bf60f83 100644
--- a/tensorflow/opensource_only.files
+++ b/tensorflow/opensource_only.files
@@ -262,7 +262,6 @@ tensorflow/third_party/toolchains/remote_config/containers.bzl
tensorflow/third_party/toolchains/remote_config/rbe_config.bzl
tensorflow/third_party/wrapt.BUILD
tensorflow/third_party/zlib.BUILD
-tensorflow/tools/build_info/BUILD
tensorflow/tools/ci_build/release/common.sh
tensorflow/tools/ci_build/release/common_win.bat
tensorflow/tools/ci_build/release/macos/cpu_libtensorflow/build.sh
@@ -393,7 +392,6 @@ tensorflow/tools/def_file_filter/BUILD
tensorflow/tools/def_file_filter/BUILD.tpl
tensorflow/tools/def_file_filter/def_file_filter.py.tpl
tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
-tensorflow/tools/git/BUILD
tensorflow/tools/lib_package/BUILD
tensorflow/tools/lib_package/LibTensorFlowTest.java
tensorflow/tools/lib_package/README.md
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index edbba09ab25..8be4c143ab8 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -26,9 +26,6 @@ load("//tensorflow:tensorflow.bzl", "tf_external_workspace_visible")
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_pybind_cc_library_wrapper")
-
-# buildifier: disable=same-origin-load
-load("//tensorflow:tensorflow.bzl", "tf_local_platform_constraint")
load("//tensorflow/core/platform:build_config.bzl", "pyx_library", "tf_additional_all_protos", "tf_additional_lib_deps", "tf_proto_library", "tf_proto_library_py", "tf_protos_grappler") # @unused
load("//tensorflow/core/platform:build_config_root.bzl", "if_static", "tf_additional_plugin_deps", "tf_additional_xla_deps_py")
load("//tensorflow/python:build_defs.bzl", "tf_gen_op_wrapper_private_py")
@@ -236,12 +233,9 @@ py_library(
],
)
-# TODO(gunan): Investigate making this action hermetic so we do not need
-# to run it locally.
tf_py_build_info_genrule(
name = "py_build_info_gen",
out = "platform/build_info.py",
- exec_compatible_with = tf_local_platform_constraint(),
)
py_library(
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 6c379887a35..d00ac4b4469 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -2504,12 +2504,7 @@ def tf_genrule_cmd_append_to_srcs(to_append):
return ("cat $(SRCS) > $(@) && " + "echo >> $(@) && " + "echo " + to_append +
" >> $(@)")
-def tf_local_platform_constraint():
- return ["@local_execution_config_platform//:platform_constraint"]
-
def tf_version_info_genrule(name, out):
- # TODO(gunan): Investigate making this action hermetic so we do not need
- # to run it locally.
native.genrule(
name = name,
srcs = [
@@ -2522,10 +2517,9 @@ def tf_version_info_genrule(name, out):
"$(location //tensorflow/tools/git:gen_git_source) --generate $(SRCS) \"$@\" --git_tag_override=$${GIT_TAG_OVERRIDE:-}",
local = 1,
exec_tools = [clean_dep("//tensorflow/tools/git:gen_git_source")],
- exec_compatible_with = tf_local_platform_constraint(),
)
-def tf_py_build_info_genrule(name, out, exec_compatible_with, **kwargs):
+def tf_py_build_info_genrule(name, out, **kwargs):
native.genrule(
name = name,
outs = [out],
diff --git a/tensorflow/tools/build_info/BUILD b/tensorflow/tools/build_info/BUILD
index eb5f909a7dd..91627d41d04 100644
--- a/tensorflow/tools/build_info/BUILD
+++ b/tensorflow/tools/build_info/BUILD
@@ -9,7 +9,6 @@ package(
py_binary(
name = "gen_build_info",
srcs = ["gen_build_info.py"],
- exec_compatible_with = ["@local_execution_config_platform//:platform_constraint"],
python_version = "PY3",
srcs_version = "PY2AND3",
deps = [
diff --git a/tensorflow/tools/git/BUILD b/tensorflow/tools/git/BUILD
index 90eb3003f5a..c1f0577f33b 100644
--- a/tensorflow/tools/git/BUILD
+++ b/tensorflow/tools/git/BUILD
@@ -11,7 +11,6 @@ package(
py_binary(
name = "gen_git_source",
srcs = ["gen_git_source.py"],
- exec_compatible_with = ["@local_execution_config_platform//:platform_constraint"],
python_version = "PY3",
srcs_version = "PY2AND3",
)
diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl
index 08ba167a998..3a0be1b0173 100644
--- a/third_party/py/BUILD.tpl
+++ b/third_party/py/BUILD.tpl
@@ -28,8 +28,6 @@ toolchain(
name = "py_toolchain",
toolchain = ":py_runtime_pair",
toolchain_type = "@bazel_tools//tools/python:toolchain_type",
- target_compatible_with = [%{PLATFORM_CONSTRAINT}],
- exec_compatible_with = [%{PLATFORM_CONSTRAINT}],
)
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl
index 2f75262ea9f..6e9a22f8063 100644
--- a/third_party/py/python_configure.bzl
+++ b/third_party/py/python_configure.bzl
@@ -240,15 +240,11 @@ def _create_local_python_repository(repository_ctx):
"numpy_include",
)
- platform_constraint = ""
- if repository_ctx.attr.platform_constraint:
- platform_constraint = "\"%s\"" % repository_ctx.attr.platform_constraint
repository_ctx.template("BUILD", build_tpl, {
"%{PYTHON_BIN_PATH}": python_bin,
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
"%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule,
- "%{PLATFORM_CONSTRAINT}": platform_constraint,
})
def _create_remote_python_repository(repository_ctx, remote_config_repo):
@@ -272,31 +268,18 @@ _ENVIRONS = [
PYTHON_LIB_PATH,
]
-local_python_configure = repository_rule(
- implementation = _create_local_python_repository,
- environ = _ENVIRONS,
- attrs = {
- "environ": attr.string_dict(),
- "platform_constraint": attr.string(),
- },
-)
-
remote_python_configure = repository_rule(
implementation = _create_local_python_repository,
environ = _ENVIRONS,
remotable = True,
attrs = {
"environ": attr.string_dict(),
- "platform_constraint": attr.string(),
},
)
python_configure = repository_rule(
implementation = _python_autoconf_impl,
environ = _ENVIRONS + [TF_PYTHON_CONFIG_REPO],
- attrs = {
- "platform_constraint": attr.string(),
- },
)
"""Detects and configures the local Python.
diff --git a/third_party/remote_config/BUILD.tpl b/third_party/remote_config/BUILD.tpl
index e7f93008431..7bcee410756 100644
--- a/third_party/remote_config/BUILD.tpl
+++ b/third_party/remote_config/BUILD.tpl
@@ -1,25 +1,8 @@
-# Each platform creates a constraint @<platform>//:platform_constraint that
-# is listed in its constraint_values; rule that want to select a specific
-# platform to run on can put @<platform>//:platform_constraing into their
-# exec_compatible_with attribute.
-# Toolchains can similarly be marked with target_compatible_with or
-# exec_compatible_with to bind them to this platform.
-constraint_setting(
- name = "platform_setting"
-)
-
-constraint_value(
- name = "platform_constraint",
- constraint_setting = ":platform_setting",
- visibility = ["//visibility:public"],
-)
-
platform(
name = "platform",
constraint_values = [
"@bazel_tools//platforms:x86_64",
"@bazel_tools//platforms:%{platform}",
- ":platform_constraint",
],
exec_properties = %{exec_properties},
)
diff --git a/third_party/toolchains/remote_config/configs.bzl b/third_party/toolchains/remote_config/configs.bzl
index c97d47f64d6..4c94abf45b3 100644
--- a/third_party/toolchains/remote_config/configs.bzl
+++ b/third_party/toolchains/remote_config/configs.bzl
@@ -1,12 +1,8 @@
"""Configurations of RBE builds used with remote config."""
-load("//third_party/toolchains/remote_config:rbe_config.bzl", "tensorflow_local_config", "tensorflow_rbe_config", "tensorflow_rbe_win_config")
+load("//third_party/toolchains/remote_config:rbe_config.bzl", "tensorflow_rbe_config", "tensorflow_rbe_win_config")
def initialize_rbe_configs():
- tensorflow_local_config(
- name = "local_execution",
- )
-
tensorflow_rbe_config(
name = "ubuntu16.04-manylinux2010-py3",
os = "ubuntu16.04-manylinux2010",
diff --git a/third_party/toolchains/remote_config/rbe_config.bzl b/third_party/toolchains/remote_config/rbe_config.bzl
index 8ab819c5328..6709cad4eb3 100644
--- a/third_party/toolchains/remote_config/rbe_config.bzl
+++ b/third_party/toolchains/remote_config/rbe_config.bzl
@@ -1,6 +1,6 @@
"""Macro that creates external repositories for remote config."""
-load("//third_party/py:python_configure.bzl", "local_python_configure", "remote_python_configure")
+load("//third_party/py:python_configure.bzl", "remote_python_configure")
load("//third_party/gpus:cuda_configure.bzl", "remote_cuda_configure")
load("//third_party/nccl:nccl_configure.bzl", "remote_nccl_configure")
load("//third_party/gpus:rocm_configure.bzl", "remote_rocm_configure")
@@ -113,7 +113,6 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
- platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
remote_rocm_configure(
@@ -128,17 +127,10 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No
"Pool": "default",
}
- remote_platform_configure(
- name = "%s_config_platform" % name,
- platform = "linux",
- platform_exec_properties = exec_properties,
- )
-
remote_python_configure(
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
- platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
else:
fail("Neither cuda_version, rocm_version nor python_version specified.")
@@ -164,19 +156,7 @@ def _tensorflow_rbe_win_config(name, python_bin_path, container_name = "windows-
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
- platform_constraint = "@%s_config_platform//:platform_constraint" % name,
- )
-
-def _tensorflow_local_config(name):
- remote_platform_configure(
- name = "%s_config_platform" % name,
- platform_exec_properties = {},
- )
- local_python_configure(
- name = "%s_config_python" % name,
- platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
tensorflow_rbe_config = _tensorflow_rbe_config
tensorflow_rbe_win_config = _tensorflow_rbe_win_config
-tensorflow_local_config = _tensorflow_local_config