experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add two repository rules:

Browse Source 
- @local_execution_config_platform: local platform to allow selecting locally
  executed tools on
- @local_execution_config_python: python configured for execution on the local
  machine during otherwise remote builds

Mark rules that are required to run locally to require our local platform.

This allows python paths to differ between the remote docker image and local
machine.

For example, the local machine might have python 3.7 installed in
/usr/bin/python3, while the remote docker should use a python installed
in /usr/local/bin/python3.8.

PiperOrigin-RevId: 307585019
Change-Id: I29313121beb967b77ae123e7d1b614c688cb40ca
This commit is contained in:
Taehee Jeong 2020-04-21 05:17:27 -07:00 committed by TensorFlower Gardener
parent 5d832c1d74
commit 0f631f8fb1
11 changed files with 3 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
WORKSPACE
View File

@ -18,8 +18,6 @@ load("//tensorflow:workspace.bzl", "tf_repositories")
# Please add all new TensorFlow dependencies in workspace.bzl.
tf_repositories()
register_execution_platforms("@local_execution_config_platform//:platform")
register_toolchains("@local_execution_config_python//:py_toolchain")
register_toolchains("@local_config_python//:py_toolchain")
load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories")

2
tensorflow/opensource_only.files
View File

@ -262,7 +262,6 @@ tensorflow/third_party/toolchains/remote_config/containers.bzl
tensorflow/third_party/toolchains/remote_config/rbe_config.bzl
tensorflow/third_party/wrapt.BUILD
tensorflow/third_party/zlib.BUILD
tensorflow/tools/build_info/BUILD
tensorflow/tools/ci_build/release/common.sh
tensorflow/tools/ci_build/release/common_win.bat
tensorflow/tools/ci_build/release/macos/cpu_libtensorflow/build.sh
@ -393,7 +392,6 @@ tensorflow/tools/def_file_filter/BUILD
tensorflow/tools/def_file_filter/BUILD.tpl
tensorflow/tools/def_file_filter/def_file_filter.py.tpl
tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
tensorflow/tools/git/BUILD
tensorflow/tools/lib_package/BUILD
tensorflow/tools/lib_package/LibTensorFlowTest.java
tensorflow/tools/lib_package/README.md

6
tensorflow/python/BUILD
View File

@ -26,9 +26,6 @@ load("//tensorflow:tensorflow.bzl", "tf_external_workspace_visible")
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_pybind_cc_library_wrapper")
# buildifier: disable=same-origin-load
load("//tensorflow:tensorflow.bzl", "tf_local_platform_constraint")
load("//tensorflow/core/platform:build_config.bzl", "pyx_library", "tf_additional_all_protos", "tf_additional_lib_deps", "tf_proto_library", "tf_proto_library_py", "tf_protos_grappler") # @unused
load("//tensorflow/core/platform:build_config_root.bzl", "if_static", "tf_additional_plugin_deps", "tf_additional_xla_deps_py")
load("//tensorflow/python:build_defs.bzl", "tf_gen_op_wrapper_private_py")
@ -236,12 +233,9 @@ py_library(
],
)
# TODO(gunan): Investigate making this action hermetic so we do not need
# to run it locally.
tf_py_build_info_genrule(
name = "py_build_info_gen",
out = "platform/build_info.py",
exec_compatible_with = tf_local_platform_constraint(),
)
py_library(

8
tensorflow/tensorflow.bzl
View File

@ -2504,12 +2504,7 @@ def tf_genrule_cmd_append_to_srcs(to_append):
return ("cat $(SRCS) > $(@) && " + "echo >> $(@) && " + "echo " + to_append +
" >> $(@)")
def tf_local_platform_constraint():
return ["@local_execution_config_platform//:platform_constraint"]
def tf_version_info_genrule(name, out):
# TODO(gunan): Investigate making this action hermetic so we do not need
# to run it locally.
native.genrule(
name = name,
srcs = [
@ -2522,10 +2517,9 @@ def tf_version_info_genrule(name, out):
"$(location //tensorflow/tools/git:gen_git_source) --generate $(SRCS) \"$@\" --git_tag_override=$${GIT_TAG_OVERRIDE:-}",
local = 1,
exec_tools = [clean_dep("//tensorflow/tools/git:gen_git_source")],
exec_compatible_with = tf_local_platform_constraint(),
)
def tf_py_build_info_genrule(name, out, exec_compatible_with, **kwargs):
def tf_py_build_info_genrule(name, out, **kwargs):
native.genrule(
name = name,
outs = [out],

1
tensorflow/tools/build_info/BUILD
View File

@ -9,7 +9,6 @@ package(
py_binary(
name = "gen_build_info",
srcs = ["gen_build_info.py"],
exec_compatible_with = ["@local_execution_config_platform//:platform_constraint"],
python_version = "PY3",
srcs_version = "PY2AND3",
deps = [

1
tensorflow/tools/git/BUILD
View File

@ -11,7 +11,6 @@ package(
py_binary(
name = "gen_git_source",
srcs = ["gen_git_source.py"],
exec_compatible_with = ["@local_execution_config_platform//:platform_constraint"],
python_version = "PY3",
srcs_version = "PY2AND3",
)

2
third_party/py/BUILD.tpl vendored
View File

@ -28,8 +28,6 @@ toolchain(
name = "py_toolchain",
toolchain = ":py_runtime_pair",
toolchain_type = "@bazel_tools//tools/python:toolchain_type",
target_compatible_with = [%{PLATFORM_CONSTRAINT}],
exec_compatible_with = [%{PLATFORM_CONSTRAINT}],
)
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib

17
third_party/py/python_configure.bzl vendored
View File

@ -240,15 +240,11 @@ def _create_local_python_repository(repository_ctx):
"numpy_include",
)
platform_constraint = ""
if repository_ctx.attr.platform_constraint:
platform_constraint = "\"%s\"" % repository_ctx.attr.platform_constraint
repository_ctx.template("BUILD", build_tpl, {
"%{PYTHON_BIN_PATH}": python_bin,
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
"%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule,
"%{PLATFORM_CONSTRAINT}": platform_constraint,
})
def _create_remote_python_repository(repository_ctx, remote_config_repo):
@ -272,31 +268,18 @@ _ENVIRONS = [
PYTHON_LIB_PATH,
]
local_python_configure = repository_rule(
implementation = _create_local_python_repository,
environ = _ENVIRONS,
attrs = {
"environ": attr.string_dict(),
"platform_constraint": attr.string(),
},
)
remote_python_configure = repository_rule(
implementation = _create_local_python_repository,
environ = _ENVIRONS,
remotable = True,
attrs = {
"environ": attr.string_dict(),
"platform_constraint": attr.string(),
},
)
python_configure = repository_rule(
implementation = _python_autoconf_impl,
environ = _ENVIRONS + [TF_PYTHON_CONFIG_REPO],
attrs = {
"platform_constraint": attr.string(),
},
)
"""Detects and configures the local Python.

17
third_party/remote_config/BUILD.tpl vendored
View File

@ -1,25 +1,8 @@
# Each platform creates a constraint @<platform>//:platform_constraint that
# is listed in its constraint_values; rule that want to select a specific
# platform to run on can put @<platform>//:platform_constraing into their
# exec_compatible_with attribute.
# Toolchains can similarly be marked with target_compatible_with or
# exec_compatible_with to bind them to this platform.
constraint_setting(
name = "platform_setting"
)
constraint_value(
name = "platform_constraint",
constraint_setting = ":platform_setting",
visibility = ["//visibility:public"],
)
platform(
name = "platform",
constraint_values = [
"@bazel_tools//platforms:x86_64",
"@bazel_tools//platforms:%{platform}",
":platform_constraint",
],
exec_properties = %{exec_properties},
)

6
third_party/toolchains/remote_config/configs.bzl vendored
View File

@ -1,12 +1,8 @@
"""Configurations of RBE builds used with remote config."""
load("//third_party/toolchains/remote_config:rbe_config.bzl", "tensorflow_local_config", "tensorflow_rbe_config", "tensorflow_rbe_win_config")
load("//third_party/toolchains/remote_config:rbe_config.bzl", "tensorflow_rbe_config", "tensorflow_rbe_win_config")
def initialize_rbe_configs():
tensorflow_local_config(
name = "local_execution",
)
tensorflow_rbe_config(
name = "ubuntu16.04-manylinux2010-py3",
os = "ubuntu16.04-manylinux2010",

22
third_party/toolchains/remote_config/rbe_config.bzl vendored
View File

@ -1,6 +1,6 @@
"""Macro that creates external repositories for remote config."""
load("//third_party/py:python_configure.bzl", "local_python_configure", "remote_python_configure")
load("//third_party/py:python_configure.bzl", "remote_python_configure")
load("//third_party/gpus:cuda_configure.bzl", "remote_cuda_configure")
load("//third_party/nccl:nccl_configure.bzl", "remote_nccl_configure")
load("//third_party/gpus:rocm_configure.bzl", "remote_rocm_configure")
@ -113,7 +113,6 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
remote_rocm_configure(
@ -128,17 +127,10 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No
"Pool": "default",
}
remote_platform_configure(
name = "%s_config_platform" % name,
platform = "linux",
platform_exec_properties = exec_properties,
)
remote_python_configure(
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
else:
fail("Neither cuda_version, rocm_version nor python_version specified.")
@ -164,19 +156,7 @@ def _tensorflow_rbe_win_config(name, python_bin_path, container_name = "windows-
name = "%s_config_python" % name,
environ = env,
exec_properties = exec_properties,
platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
def _tensorflow_local_config(name):
remote_platform_configure(
name = "%s_config_platform" % name,
platform_exec_properties = {},
)
local_python_configure(
name = "%s_config_python" % name,
platform_constraint = "@%s_config_platform//:platform_constraint" % name,
)
tensorflow_rbe_config = _tensorflow_rbe_config
tensorflow_rbe_win_config = _tensorflow_rbe_win_config
tensorflow_local_config = _tensorflow_local_config