diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files index 3053e7c943c..d4df3df079e 100644 --- a/tensorflow/opensource_only.files +++ b/tensorflow/opensource_only.files @@ -263,6 +263,7 @@ tensorflow/third_party/toolchains/remote_config/containers.bzl tensorflow/third_party/toolchains/remote_config/rbe_config.bzl tensorflow/third_party/wrapt.BUILD tensorflow/third_party/zlib.BUILD +tensorflow/tools/build_info/BUILD tensorflow/tools/ci_build/release/common.sh tensorflow/tools/ci_build/release/common_win.bat tensorflow/tools/ci_build/release/macos/cpu_libtensorflow/build.sh @@ -393,6 +394,7 @@ tensorflow/tools/def_file_filter/BUILD tensorflow/tools/def_file_filter/BUILD.tpl tensorflow/tools/def_file_filter/def_file_filter.py.tpl tensorflow/tools/def_file_filter/def_file_filter_configure.bzl +tensorflow/tools/git/BUILD tensorflow/tools/lib_package/BUILD tensorflow/tools/lib_package/LibTensorFlowTest.java tensorflow/tools/lib_package/README.md diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index aab2fcd0411..7d7a7ea7fa5 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -26,6 +26,9 @@ load("//tensorflow:tensorflow.bzl", "tf_external_workspace_visible") # buildifier: disable=same-origin-load load("//tensorflow:tensorflow.bzl", "tf_pybind_cc_library_wrapper") + +# buildifier: disable=same-origin-load +load("//tensorflow:tensorflow.bzl", "tf_local_platform_constraint") load("//tensorflow/core/platform:build_config.bzl", "pyx_library", "tf_additional_all_protos", "tf_additional_lib_deps", "tf_proto_library", "tf_proto_library_py", "tf_protos_grappler") # @unused load("//tensorflow/core/platform:build_config_root.bzl", "if_static", "tf_additional_plugin_deps", "tf_additional_xla_deps_py") load("//tensorflow/python:build_defs.bzl", "tf_gen_op_wrapper_private_py") @@ -233,9 +236,12 @@ py_library( ], ) +# TODO(gunan): Investigate making this action hermetic so we do not need +# to run it locally. tf_py_build_info_genrule( name = "py_build_info_gen", out = "platform/build_info.py", + exec_compatible_with = tf_local_platform_constraint(), ) py_library( diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 246cbe86025..4070578efad 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -2522,7 +2522,12 @@ def tf_genrule_cmd_append_to_srcs(to_append): return ("cat $(SRCS) > $(@) && " + "echo >> $(@) && " + "echo " + to_append + " >> $(@)") +def tf_local_platform_constraint(): + return ["@local_execution_config_platform//:platform_constraint"] + def tf_version_info_genrule(name, out): + # TODO(gunan): Investigate making this action hermetic so we do not need + # to run it locally. native.genrule( name = name, srcs = [ @@ -2535,9 +2540,10 @@ def tf_version_info_genrule(name, out): "$(location //tensorflow/tools/git:gen_git_source) --generate $(SRCS) \"$@\" --git_tag_override=$${GIT_TAG_OVERRIDE:-}", local = 1, exec_tools = [clean_dep("//tensorflow/tools/git:gen_git_source")], + exec_compatible_with = tf_local_platform_constraint(), ) -def tf_py_build_info_genrule(name, out, **kwargs): +def tf_py_build_info_genrule(name, out, exec_compatible_with, **kwargs): native.genrule( name = name, outs = [out], diff --git a/tensorflow/tools/build_info/BUILD b/tensorflow/tools/build_info/BUILD index 91627d41d04..eb5f909a7dd 100644 --- a/tensorflow/tools/build_info/BUILD +++ b/tensorflow/tools/build_info/BUILD @@ -9,6 +9,7 @@ package( py_binary( name = "gen_build_info", srcs = ["gen_build_info.py"], + exec_compatible_with = ["@local_execution_config_platform//:platform_constraint"], python_version = "PY3", srcs_version = "PY2AND3", deps = [ diff --git a/tensorflow/tools/git/BUILD b/tensorflow/tools/git/BUILD index c1f0577f33b..90eb3003f5a 100644 --- a/tensorflow/tools/git/BUILD +++ b/tensorflow/tools/git/BUILD @@ -11,6 +11,7 @@ package( py_binary( name = "gen_git_source", srcs = ["gen_git_source.py"], + exec_compatible_with = ["@local_execution_config_platform//:platform_constraint"], python_version = "PY3", srcs_version = "PY2AND3", ) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index a8f4806c130..c810f21339b 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -77,10 +77,18 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): tf_repositories(path_prefix, tf_repo_name) tf_bind() +# Toolchains & platforms required by Tensorflow to build. +def tf_toolchains(): + native.register_execution_platforms("@local_execution_config_platform//:platform") + native.register_toolchains("@local_execution_config_python//:py_toolchain") + # Define all external repositories required by TensorFlow def tf_repositories(path_prefix = "", tf_repo_name = ""): """All external dependencies for TF builds.""" + # Initialize toolchains and platforms. + tf_toolchains() + # Loads all external repos to configure RBE builds. initialize_rbe_configs() diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl index 3a0be1b0173..08ba167a998 100644 --- a/third_party/py/BUILD.tpl +++ b/third_party/py/BUILD.tpl @@ -28,6 +28,8 @@ toolchain( name = "py_toolchain", toolchain = ":py_runtime_pair", toolchain_type = "@bazel_tools//tools/python:toolchain_type", + target_compatible_with = [%{PLATFORM_CONSTRAINT}], + exec_compatible_with = [%{PLATFORM_CONSTRAINT}], ) # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index 6e9a22f8063..2f75262ea9f 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -240,11 +240,15 @@ def _create_local_python_repository(repository_ctx): "numpy_include", ) + platform_constraint = "" + if repository_ctx.attr.platform_constraint: + platform_constraint = "\"%s\"" % repository_ctx.attr.platform_constraint repository_ctx.template("BUILD", build_tpl, { "%{PYTHON_BIN_PATH}": python_bin, "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, "%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule, + "%{PLATFORM_CONSTRAINT}": platform_constraint, }) def _create_remote_python_repository(repository_ctx, remote_config_repo): @@ -268,18 +272,31 @@ _ENVIRONS = [ PYTHON_LIB_PATH, ] +local_python_configure = repository_rule( + implementation = _create_local_python_repository, + environ = _ENVIRONS, + attrs = { + "environ": attr.string_dict(), + "platform_constraint": attr.string(), + }, +) + remote_python_configure = repository_rule( implementation = _create_local_python_repository, environ = _ENVIRONS, remotable = True, attrs = { "environ": attr.string_dict(), + "platform_constraint": attr.string(), }, ) python_configure = repository_rule( implementation = _python_autoconf_impl, environ = _ENVIRONS + [TF_PYTHON_CONFIG_REPO], + attrs = { + "platform_constraint": attr.string(), + }, ) """Detects and configures the local Python. diff --git a/third_party/remote_config/BUILD.tpl b/third_party/remote_config/BUILD.tpl index 7bcee410756..e7f93008431 100644 --- a/third_party/remote_config/BUILD.tpl +++ b/third_party/remote_config/BUILD.tpl @@ -1,8 +1,25 @@ +# Each platform creates a constraint @//:platform_constraint that +# is listed in its constraint_values; rule that want to select a specific +# platform to run on can put @//:platform_constraing into their +# exec_compatible_with attribute. +# Toolchains can similarly be marked with target_compatible_with or +# exec_compatible_with to bind them to this platform. +constraint_setting( + name = "platform_setting" +) + +constraint_value( + name = "platform_constraint", + constraint_setting = ":platform_setting", + visibility = ["//visibility:public"], +) + platform( name = "platform", constraint_values = [ "@bazel_tools//platforms:x86_64", "@bazel_tools//platforms:%{platform}", + ":platform_constraint", ], exec_properties = %{exec_properties}, ) diff --git a/third_party/remote_config/remote_platform_configure.bzl b/third_party/remote_config/remote_platform_configure.bzl index 5c2918b8e84..1031fbfb2d9 100644 --- a/third_party/remote_config/remote_platform_configure.bzl +++ b/third_party/remote_config/remote_platform_configure.bzl @@ -2,6 +2,14 @@ def _remote_platform_configure_impl(repository_ctx): platform = repository_ctx.attr.platform + if platform == "local": + os = repository_ctx.os.name.lower() + if os.startswith("windows"): + platform = "windows" + elif os.startswith("mac os"): + platform = "osx" + else: + platform = "linux" exec_properties = repository_ctx.attr.platform_exec_properties serialized_exec_properties = "{" @@ -22,6 +30,6 @@ remote_platform_configure = repository_rule( implementation = _remote_platform_configure_impl, attrs = { "platform_exec_properties": attr.string_dict(mandatory = True), - "platform": attr.string(default = "linux", values = ["linux", "windows"]), + "platform": attr.string(default = "linux", values = ["linux", "windows", "local"]), }, ) diff --git a/third_party/toolchains/remote_config/configs.bzl b/third_party/toolchains/remote_config/configs.bzl index 4c94abf45b3..c97d47f64d6 100644 --- a/third_party/toolchains/remote_config/configs.bzl +++ b/third_party/toolchains/remote_config/configs.bzl @@ -1,8 +1,12 @@ """Configurations of RBE builds used with remote config.""" -load("//third_party/toolchains/remote_config:rbe_config.bzl", "tensorflow_rbe_config", "tensorflow_rbe_win_config") +load("//third_party/toolchains/remote_config:rbe_config.bzl", "tensorflow_local_config", "tensorflow_rbe_config", "tensorflow_rbe_win_config") def initialize_rbe_configs(): + tensorflow_local_config( + name = "local_execution", + ) + tensorflow_rbe_config( name = "ubuntu16.04-manylinux2010-py3", os = "ubuntu16.04-manylinux2010", diff --git a/third_party/toolchains/remote_config/rbe_config.bzl b/third_party/toolchains/remote_config/rbe_config.bzl index 6709cad4eb3..597ab558eb7 100644 --- a/third_party/toolchains/remote_config/rbe_config.bzl +++ b/third_party/toolchains/remote_config/rbe_config.bzl @@ -1,6 +1,6 @@ """Macro that creates external repositories for remote config.""" -load("//third_party/py:python_configure.bzl", "remote_python_configure") +load("//third_party/py:python_configure.bzl", "local_python_configure", "remote_python_configure") load("//third_party/gpus:cuda_configure.bzl", "remote_cuda_configure") load("//third_party/nccl:nccl_configure.bzl", "remote_nccl_configure") load("//third_party/gpus:rocm_configure.bzl", "remote_rocm_configure") @@ -113,6 +113,7 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No name = "%s_config_python" % name, environ = env, exec_properties = exec_properties, + platform_constraint = "@%s_config_platform//:platform_constraint" % name, ) remote_rocm_configure( @@ -127,10 +128,17 @@ def _tensorflow_rbe_config(name, compiler, python_version, os, rocm_version = No "Pool": "default", } + remote_platform_configure( + name = "%s_config_platform" % name, + platform = "linux", + platform_exec_properties = exec_properties, + ) + remote_python_configure( name = "%s_config_python" % name, environ = env, exec_properties = exec_properties, + platform_constraint = "@%s_config_platform//:platform_constraint" % name, ) else: fail("Neither cuda_version, rocm_version nor python_version specified.") @@ -156,7 +164,20 @@ def _tensorflow_rbe_win_config(name, python_bin_path, container_name = "windows- name = "%s_config_python" % name, environ = env, exec_properties = exec_properties, + platform_constraint = "@%s_config_platform//:platform_constraint" % name, + ) + +def _tensorflow_local_config(name): + remote_platform_configure( + name = "%s_config_platform" % name, + platform = "local", + platform_exec_properties = {}, + ) + local_python_configure( + name = "%s_config_python" % name, + platform_constraint = "@%s_config_platform//:platform_constraint" % name, ) tensorflow_rbe_config = _tensorflow_rbe_config tensorflow_rbe_win_config = _tensorflow_rbe_win_config +tensorflow_local_config = _tensorflow_local_config