Merge pull request #44489 from perfinion/r2.4-syslib-cherrypick

systemlib cherrypicks for r2.4

.bazelrc

@@ -296,9 +296,11 @@ build:windows --host_copt=/D_USE_MATH_DEFINES
 build:linux --define=PREFIX=/usr
 build:linux --define=LIBDIR=$(PREFIX)/lib
 build:linux --define=INCLUDEDIR=$(PREFIX)/include
+build:linux --define=PROTOBUF_INCLUDE_PATH=$(PREFIX)/include
 build:macos --define=PREFIX=/usr
 build:macos --define=LIBDIR=$(PREFIX)/lib
 build:macos --define=INCLUDEDIR=$(PREFIX)/include
+build:macos --define=PROTOBUF_INCLUDE_PATH=$(PREFIX)/include
 # TF_SYSTEM_LIBS do not work on windows.
 
 # By default, build TF in C++ 14 mode.

WORKSPACE

@@ -113,26 +113,10 @@ http_archive(
 # Required for dependency @com_github_grpc_grpc
 
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
-
 grpc_deps()
 
-load(
-    "@build_bazel_rules_apple//apple:repositories.bzl",
-    "apple_rules_dependencies",
-)
-
-apple_rules_dependencies()
-
-load(
-    "@build_bazel_apple_support//lib:repositories.bzl",
-    "apple_support_dependencies",
-)
-
-apple_support_dependencies()
-
-load("@upb//bazel:repository_defs.bzl", "bazel_version_repository")
-
-bazel_version_repository(name = "bazel_version")
+load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
+grpc_extra_deps()
 
 load("//third_party/googleapis:repository_rules.bzl", "config_googleapis")
 

configure.py

@@ -1163,12 +1163,9 @@ def set_system_libs_flag(environ_cp):
       syslibs = ','.join(sorted(syslibs.split()))
     write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs)
 
-  if 'PREFIX' in environ_cp:
-    write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX'])
-  if 'LIBDIR' in environ_cp:
-    write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR'])
-  if 'INCLUDEDIR' in environ_cp:
-    write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR'])
+  for varname in ('PREFIX', 'LIBDIR', 'INCLUDEDIR', 'PROTOBUF_INCLUDE_PATH'):
+    if varname in environ_cp:
+      write_to_bazelrc('build --define=%s=%s' % (varname, environ_cp[varname]))
 
 
 def is_reduced_optimize_huge_functions_available(environ_cp):

tensorflow/workspace.bzl

@@ -545,6 +545,7 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "dill_archive",
         build_file = clean_dep("//third_party:dill.BUILD"),
+        system_build_file = clean_dep("//third_party/systemlibs:dill.BUILD"),
         urls = [
             "http://mirror.tensorflow.org/files.pythonhosted.org/packages/c7/11/345f3173809cea7f1a193bfbf02403fff250a3360e0e118a1630985e547d/dill-0.3.1.1.tar.gz",
             "https://files.pythonhosted.org/packages/c7/11/345f3173809cea7f1a193bfbf02403fff250a3360e0e118a1630985e547d/dill-0.3.1.1.tar.gz",
@@ -556,6 +557,7 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
     tf_http_archive(
         name = "tblib_archive",
         build_file = clean_dep("//third_party:tblib.BUILD"),
+        system_build_file = clean_dep("//third_party/systemlibs:tblib.BUILD"),
         urls = [
             "http://mirror.tensorflow.org/files.pythonhosted.org/packages/ec/c4/8c651f3240a73c28a218194f3d527eb2be5a173d08501060cdee84ade33f/tblib-1.3.2.tar.gz",
             "https://files.pythonhosted.org/packages/ec/c4/8c651f3240a73c28a218194f3d527eb2be5a173d08501060cdee84ade33f/tblib-1.3.2.tar.gz",
@@ -656,6 +658,10 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
             "//third_party/systemlibs:BUILD": "bazel/BUILD",
             "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD",
             "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl",
+            "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl",
+            "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl",
+            "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl",
+            "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl",
         },
         urls = [
             "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz",

third_party/systemlibs/dill.BUILD

@@ -0,0 +1,12 @@
+licenses(["notice"])  # BSD 3-clause
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "dill",
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)

third_party/systemlibs/grpc.BUILD

@@ -7,25 +7,47 @@ filegroup(
 
 cc_library(
     name = "grpc",
-    linkopts = ["-lgrpc"],
+    linkopts = [
+        "-lgrpc",
+        "-lgpr",
+    ],
     visibility = ["//visibility:public"],
 )
 
 cc_library(
     name = "grpc++",
-    linkopts = ["-lgrpc++"],
+    linkopts = [
+        "-lgrpc++",
+        "-lgpr",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "grpc++_public_hdrs",
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "grpc++_codegen_proto",
     visibility = ["//visibility:public"],
 )
 
 cc_library(
     name = "grpc_unsecure",
-    linkopts = ["-lgrpc_unsecure"],
+    linkopts = [
+        "-lgrpc_unsecure",
+        "-lgpr",
+    ],
     visibility = ["//visibility:public"],
 )
 
 cc_library(
     name = "grpc++_unsecure",
-    linkopts = ["-lgrpc++_unsecure"],
+    linkopts = [
+        "-lgrpc++_unsecure",
+        "-lgpr",
+    ],
     visibility = ["//visibility:public"],
 )
 

third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl

@@ -0,0 +1,105 @@
+"""Generates and compiles C++ grpc stubs from proto_library rules."""
+
+load("@com_github_grpc_grpc//bazel:generate_cc.bzl", "generate_cc")
+load("@com_github_grpc_grpc//bazel:protobuf.bzl", "well_known_proto_libs")
+
+def cc_grpc_library(
+        name,
+        srcs,
+        deps,
+        proto_only = False,
+        well_known_protos = False,
+        generate_mocks = False,
+        use_external = False,
+        grpc_only = False,
+        **kwargs):
+    """Generates C++ grpc classes for services defined in a proto file.
+
+    If grpc_only is True, this rule is compatible with proto_library and
+    cc_proto_library native rules such that it expects proto_library target
+    as srcs argument and generates only grpc library classes, expecting
+    protobuf messages classes library (cc_proto_library target) to be passed in
+    deps argument. By default grpc_only is False which makes this rule to behave
+    in a backwards-compatible mode (trying to generate both proto and grpc
+    classes).
+
+    Assumes the generated classes will be used in cc_api_version = 2.
+
+    Args:
+        name (str): Name of rule.
+        srcs (list): A single .proto file which contains services definitions,
+          or if grpc_only parameter is True, a single proto_library which
+          contains services descriptors.
+        deps (list): A list of C++ proto_library (or cc_proto_library) which
+          provides the compiled code of any message that the services depend on.
+        proto_only (bool): If True, create only C++ proto classes library,
+          avoid creating C++ grpc classes library (expect it in deps).
+          Deprecated, use native cc_proto_library instead. False by default.
+        well_known_protos (bool): Should this library additionally depend on
+          well known protos. Deprecated, the well known protos should be
+          specified as explicit dependencies of the proto_library target
+          (passed in srcs parameter) instead. False by default.
+        generate_mocks (bool): when True, Google Mock code for client stub is
+          generated. False by default.
+        use_external (bool): Not used.
+        grpc_only (bool): if True, generate only grpc library, expecting
+          protobuf messages library (cc_proto_library target) to be passed as
+          deps. False by default (will become True by default eventually).
+        **kwargs: rest of arguments, e.g., compatible_with and visibility
+    """
+    if len(srcs) > 1:
+        fail("Only one srcs value supported", "srcs")
+    if grpc_only and proto_only:
+        fail("A mutualy exclusive configuration is specified: grpc_only = True and proto_only = True")
+
+    extra_deps = []
+    proto_targets = []
+
+    if not grpc_only:
+        proto_target = "_" + name + "_only"
+        cc_proto_target = name if proto_only else "_" + name + "_cc_proto"
+
+        proto_deps = ["_" + dep + "_only" for dep in deps if dep.find(":") == -1]
+        proto_deps += [dep.split(":")[0] + ":" + "_" + dep.split(":")[1] + "_only" for dep in deps if dep.find(":") != -1]
+        if well_known_protos:
+            proto_deps += well_known_proto_libs()
+
+        native.proto_library(
+            name = proto_target,
+            srcs = srcs,
+            deps = proto_deps,
+            **kwargs
+        )
+
+        native.cc_proto_library(
+            name = cc_proto_target,
+            deps = [":" + proto_target],
+            **kwargs
+        )
+        extra_deps.append(":" + cc_proto_target)
+        proto_targets.append(proto_target)
+    else:
+        if not srcs:
+            fail("srcs cannot be empty", "srcs")
+        proto_targets += srcs
+
+    if not proto_only:
+        codegen_grpc_target = "_" + name + "_grpc_codegen"
+        generate_cc(
+            name = codegen_grpc_target,
+            srcs = proto_targets,
+            plugin = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin",
+            well_known_protos = well_known_protos,
+            generate_mocks = generate_mocks,
+            **kwargs
+        )
+
+        native.cc_library(
+            name = name,
+            srcs = [":" + codegen_grpc_target],
+            hdrs = [":" + codegen_grpc_target],
+            deps = deps +
+                   extra_deps +
+                   ["@com_github_grpc_grpc//:grpc++_codegen_proto"],
+            **kwargs
+        )

third_party/systemlibs/grpc.bazel.generate_cc.bzl

@@ -0,0 +1,186 @@
+"""Generates C++ grpc stubs from proto_library rules.
+
+This is an internal rule used by cc_grpc_library, and shouldn't be used
+directly.
+"""
+
+load(
+    "@com_github_grpc_grpc//bazel:protobuf.bzl",
+    "get_include_directory",
+    "get_plugin_args",
+    "get_proto_root",
+    "proto_path_to_generated_filename",
+)
+
+_GRPC_PROTO_HEADER_FMT = "{}.grpc.pb.h"
+_GRPC_PROTO_SRC_FMT = "{}.grpc.pb.cc"
+_GRPC_PROTO_MOCK_HEADER_FMT = "{}_mock.grpc.pb.h"
+_PROTO_HEADER_FMT = "{}.pb.h"
+_PROTO_SRC_FMT = "{}.pb.cc"
+
+def _strip_package_from_path(label_package, file):
+    prefix_len = 0
+    if not file.is_source and file.path.startswith(file.root.path):
+        prefix_len = len(file.root.path) + 1
+
+    path = file.path
+    if len(label_package) == 0:
+        return path
+    if not path.startswith(label_package + "/", prefix_len):
+        fail("'{}' does not lie within '{}'.".format(path, label_package))
+    return path[prefix_len + len(label_package + "/"):]
+
+def _get_srcs_file_path(file):
+    if not file.is_source and file.path.startswith(file.root.path):
+        return file.path[len(file.root.path) + 1:]
+    return file.path
+
+def _join_directories(directories):
+    massaged_directories = [directory for directory in directories if len(directory) != 0]
+    return "/".join(massaged_directories)
+
+def generate_cc_impl(ctx):
+    """Implementation of the generate_cc rule."""
+    protos = [f for src in ctx.attr.srcs for f in src[ProtoInfo].check_deps_sources.to_list()]
+    includes = [
+        f
+        for src in ctx.attr.srcs
+        for f in src[ProtoInfo].transitive_imports.to_list()
+    ]
+    outs = []
+    proto_root = get_proto_root(
+        ctx.label.workspace_root,
+    )
+
+    label_package = _join_directories([ctx.label.workspace_root, ctx.label.package])
+    if ctx.executable.plugin:
+        outs += [
+            proto_path_to_generated_filename(
+                _strip_package_from_path(label_package, proto),
+                _GRPC_PROTO_HEADER_FMT,
+            )
+            for proto in protos
+        ]
+        outs += [
+            proto_path_to_generated_filename(
+                _strip_package_from_path(label_package, proto),
+                _GRPC_PROTO_SRC_FMT,
+            )
+            for proto in protos
+        ]
+        if ctx.attr.generate_mocks:
+            outs += [
+                proto_path_to_generated_filename(
+                    _strip_package_from_path(label_package, proto),
+                    _GRPC_PROTO_MOCK_HEADER_FMT,
+                )
+                for proto in protos
+            ]
+    else:
+        outs += [
+            proto_path_to_generated_filename(
+                _strip_package_from_path(label_package, proto),
+                _PROTO_HEADER_FMT,
+            )
+            for proto in protos
+        ]
+        outs += [
+            proto_path_to_generated_filename(
+                _strip_package_from_path(label_package, proto),
+                _PROTO_SRC_FMT,
+            )
+            for proto in protos
+        ]
+    out_files = [ctx.actions.declare_file(out) for out in outs]
+    dir_out = str(ctx.genfiles_dir.path + proto_root)
+
+    arguments = []
+    if ctx.executable.plugin:
+        arguments += get_plugin_args(
+            ctx.executable.plugin,
+            ctx.attr.flags,
+            dir_out,
+            ctx.attr.generate_mocks,
+        )
+        tools = [ctx.executable.plugin]
+    else:
+        arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out]
+        tools = []
+
+    arguments += [
+        "--proto_path={}".format(get_include_directory(i))
+        for i in includes
+    ]
+
+    # Include the output directory so that protoc puts the generated code in the
+    # right directory.
+    arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)]
+    arguments += [_get_srcs_file_path(proto) for proto in protos]
+
+    # create a list of well known proto files if the argument is non-None
+    well_known_proto_files = []
+    if ctx.attr.well_known_protos:
+        f = ctx.attr.well_known_protos.files.to_list()[0].dirname
+        if f != "external/com_google_protobuf/src/google/protobuf":
+            print(
+                "Error: Only @com_google_protobuf//:well_known_protos is supported",
+            )
+        else:
+            # f points to "external/com_google_protobuf/src/google/protobuf"
+            # add -I argument to protoc so it knows where to look for the proto files.
+            arguments += ["-I{0}".format(f + "/../..")]
+            well_known_proto_files = [
+                f
+                for f in ctx.attr.well_known_protos.files.to_list()
+            ]
+
+    ctx.actions.run(
+        inputs = protos + includes + well_known_proto_files,
+        tools = tools,
+        outputs = out_files,
+        executable = ctx.executable._protoc,
+        arguments = arguments,
+    )
+
+    return struct(files = depset(out_files))
+
+_generate_cc = rule(
+    attrs = {
+        "srcs": attr.label_list(
+            mandatory = True,
+            allow_empty = False,
+            providers = [ProtoInfo],
+        ),
+        "plugin": attr.label(
+            executable = True,
+            providers = ["files_to_run"],
+            cfg = "host",
+        ),
+        "flags": attr.string_list(
+            mandatory = False,
+            allow_empty = True,
+        ),
+        "well_known_protos": attr.label(mandatory = False),
+        "generate_mocks": attr.bool(
+            default = False,
+            mandatory = False,
+        ),
+        "_protoc": attr.label(
+            default = Label("@com_google_protobuf//:protoc"),
+            executable = True,
+            cfg = "host",
+        ),
+    },
+    # We generate .h files, so we need to output to genfiles.
+    output_to_genfiles = True,
+    implementation = generate_cc_impl,
+)
+
+def generate_cc(well_known_protos, **kwargs):
+    if well_known_protos:
+        _generate_cc(
+            well_known_protos = "@com_google_protobuf//:well_known_protos",
+            **kwargs
+        )
+    else:
+        _generate_cc(**kwargs)

third_party/systemlibs/grpc.bazel.grpc_extra_deps.bzl

@@ -0,0 +1,4 @@
+"""Stub version of @com_github_grpc_grpc//bazel:grpc_extra_deps.bzl necessary for TF system libs"""
+
+def grpc_extra_deps():
+    pass

third_party/systemlibs/grpc.bazel.protobuf.bzl

@@ -0,0 +1,244 @@
+"""Utility functions for generating protobuf code."""
+
+_PROTO_EXTENSION = ".proto"
+_VIRTUAL_IMPORTS = "/_virtual_imports/"
+
+def well_known_proto_libs():
+    return [
+        "@com_google_protobuf//:any_proto",
+        "@com_google_protobuf//:api_proto",
+        "@com_google_protobuf//:compiler_plugin_proto",
+        "@com_google_protobuf//:descriptor_proto",
+        "@com_google_protobuf//:duration_proto",
+        "@com_google_protobuf//:empty_proto",
+        "@com_google_protobuf//:field_mask_proto",
+        "@com_google_protobuf//:source_context_proto",
+        "@com_google_protobuf//:struct_proto",
+        "@com_google_protobuf//:timestamp_proto",
+        "@com_google_protobuf//:type_proto",
+        "@com_google_protobuf//:wrappers_proto",
+    ]
+
+def get_proto_root(workspace_root):
+    """Gets the root protobuf directory.
+
+    Args:
+      workspace_root: context.label.workspace_root
+
+    Returns:
+      The directory relative to which generated include paths should be.
+    """
+    if workspace_root:
+        return "/{}".format(workspace_root)
+    else:
+        return ""
+
+def _strip_proto_extension(proto_filename):
+    if not proto_filename.endswith(_PROTO_EXTENSION):
+        fail('"{}" does not end with "{}"'.format(
+            proto_filename,
+            _PROTO_EXTENSION,
+        ))
+    return proto_filename[:-len(_PROTO_EXTENSION)]
+
+def proto_path_to_generated_filename(proto_path, fmt_str):
+    """Calculates the name of a generated file for a protobuf path.
+
+    For example, "examples/protos/helloworld.proto" might map to
+      "helloworld.pb.h".
+
+    Args:
+      proto_path: The path to the .proto file.
+      fmt_str: A format string used to calculate the generated filename. For
+        example, "{}.pb.h" might be used to calculate a C++ header filename.
+
+    Returns:
+      The generated filename.
+    """
+    return fmt_str.format(_strip_proto_extension(proto_path))
+
+def get_include_directory(source_file):
+    """Returns the include directory path for the source_file.
+
+    I.e. all of the include statements within the given source_file
+    are calculated relative to the directory returned by this method.
+
+    The returned directory path can be used as the "--proto_path=" argument
+    value.
+
+    Args:
+      source_file: A proto file.
+
+    Returns:
+      The include directory path for the source_file.
+    """
+    directory = source_file.path
+    prefix_len = 0
+
+    if is_in_virtual_imports(source_file):
+        root, relative = source_file.path.split(_VIRTUAL_IMPORTS, 2)
+        result = root + _VIRTUAL_IMPORTS + relative.split("/", 1)[0]
+        return result
+
+    if not source_file.is_source and directory.startswith(source_file.root.path):
+        prefix_len = len(source_file.root.path) + 1
+
+    if directory.startswith("external", prefix_len):
+        external_separator = directory.find("/", prefix_len)
+        repository_separator = directory.find("/", external_separator + 1)
+        return directory[:repository_separator]
+    else:
+        return source_file.root.path if source_file.root.path else "."
+
+def get_plugin_args(
+        plugin,
+        flags,
+        dir_out,
+        generate_mocks,
+        plugin_name = "PLUGIN"):
+    """Returns arguments configuring protoc to use a plugin for a language.
+
+    Args:
+      plugin: An executable file to run as the protoc plugin.
+      flags: The plugin flags to be passed to protoc.
+      dir_out: The output directory for the plugin.
+      generate_mocks: A bool indicating whether to generate mocks.
+      plugin_name: A name of the plugin, it is required to be unique when there
+      are more than one plugin used in a single protoc command.
+    Returns:
+      A list of protoc arguments configuring the plugin.
+    """
+    augmented_flags = list(flags)
+    if generate_mocks:
+        augmented_flags.append("generate_mock_code=true")
+
+    augmented_dir_out = dir_out
+    if augmented_flags:
+        augmented_dir_out = ",".join(augmented_flags) + ":" + dir_out
+
+    return [
+        "--plugin=protoc-gen-{plugin_name}={plugin_path}".format(
+            plugin_name = plugin_name,
+            plugin_path = plugin.path,
+        ),
+        "--{plugin_name}_out={dir_out}".format(
+            plugin_name = plugin_name,
+            dir_out = augmented_dir_out,
+        ),
+    ]
+
+def _get_staged_proto_file(context, source_file):
+    if source_file.dirname == context.label.package or \
+       is_in_virtual_imports(source_file):
+        return source_file
+    else:
+        copied_proto = context.actions.declare_file(source_file.basename)
+        context.actions.run_shell(
+            inputs = [source_file],
+            outputs = [copied_proto],
+            command = "cp {} {}".format(source_file.path, copied_proto.path),
+            mnemonic = "CopySourceProto",
+        )
+        return copied_proto
+
+def protos_from_context(context):
+    """Copies proto files to the appropriate location.
+
+    Args:
+      context: The ctx object for the rule.
+
+    Returns:
+      A list of the protos.
+    """
+    protos = []
+    for src in context.attr.deps:
+        for file in src[ProtoInfo].direct_sources:
+            protos.append(_get_staged_proto_file(context, file))
+    return protos
+
+def includes_from_deps(deps):
+    """Get includes from rule dependencies."""
+    return [
+        file
+        for src in deps
+        for file in src[ProtoInfo].transitive_imports.to_list()
+    ]
+
+def get_proto_arguments(protos, genfiles_dir_path):
+    """Get the protoc arguments specifying which protos to compile."""
+    arguments = []
+    for proto in protos:
+        strip_prefix_len = 0
+        if is_in_virtual_imports(proto):
+            incl_directory = get_include_directory(proto)
+            if proto.path.startswith(incl_directory):
+                strip_prefix_len = len(incl_directory) + 1
+        elif proto.path.startswith(genfiles_dir_path):
+            strip_prefix_len = len(genfiles_dir_path) + 1
+
+        arguments.append(proto.path[strip_prefix_len:])
+
+    return arguments
+
+def declare_out_files(protos, context, generated_file_format):
+    """Declares and returns the files to be generated."""
+
+    out_file_paths = []
+    for proto in protos:
+        if not is_in_virtual_imports(proto):
+            out_file_paths.append(proto.basename)
+        else:
+            path = proto.path[proto.path.index(_VIRTUAL_IMPORTS) + 1:]
+            out_file_paths.append(path)
+
+    return [
+        context.actions.declare_file(
+            proto_path_to_generated_filename(
+                out_file_path,
+                generated_file_format,
+            ),
+        )
+        for out_file_path in out_file_paths
+    ]
+
+def get_out_dir(protos, context):
+    """ Returns the calculated value for --<lang>_out= protoc argument based on
+    the input source proto files and current context.
+
+    Args:
+        protos: A list of protos to be used as source files in protoc command
+        context: A ctx object for the rule.
+    Returns:
+        The value of --<lang>_out= argument.
+    """
+    at_least_one_virtual = 0
+    for proto in protos:
+        if is_in_virtual_imports(proto):
+            at_least_one_virtual = True
+        elif at_least_one_virtual:
+            fail("Proto sources must be either all virtual imports or all real")
+    if at_least_one_virtual:
+        out_dir = get_include_directory(protos[0])
+        ws_root = protos[0].owner.workspace_root
+        if ws_root and out_dir.find(ws_root) >= 0:
+            out_dir = "".join(out_dir.rsplit(ws_root, 1))
+        return struct(
+            path = out_dir,
+            import_path = out_dir[out_dir.find(_VIRTUAL_IMPORTS) + 1:],
+        )
+    return struct(path = context.genfiles_dir.path, import_path = None)
+
+def is_in_virtual_imports(source_file, virtual_folder = _VIRTUAL_IMPORTS):
+    """Determines if source_file is virtual (is placed in _virtual_imports
+    subdirectory). The output of all proto_library targets which use
+    import_prefix  and/or strip_import_prefix arguments is placed under
+    _virtual_imports directory.
+
+    Args:
+        source_file: A proto file.
+        virtual_folder: The virtual folder name (is set to "_virtual_imports"
+            by default)
+    Returns:
+        True if source_file is located under _virtual_imports, False otherwise.
+    """
+    return not source_file.is_source and virtual_folder in source_file.path

third_party/systemlibs/protobuf.BUILD

@@ -12,29 +12,33 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 
-PROTO_FILES = [
-    "google/protobuf/any.proto",
-    "google/protobuf/api.proto",
-    "google/protobuf/compiler/plugin.proto",
-    "google/protobuf/descriptor.proto",
-    "google/protobuf/duration.proto",
-    "google/protobuf/empty.proto",
-    "google/protobuf/field_mask.proto",
-    "google/protobuf/source_context.proto",
-    "google/protobuf/struct.proto",
-    "google/protobuf/timestamp.proto",
-    "google/protobuf/type.proto",
-    "google/protobuf/wrappers.proto",
-]
+# Map of all well known protos.
+# name => (include path, imports)
+WELL_KNOWN_PROTO_MAP = {
+    "any" : ("google/protobuf/any.proto", []),
+    "api" : ("google/protobuf/api.proto", ["source_context", "type"]),
+    "compiler_plugin" : ("google/protobuf/compiler/plugin.proto", ["descriptor"]),
+    "descriptor" : ("google/protobuf/descriptor.proto", []),
+    "duration" : ("google/protobuf/duration.proto", []),
+    "empty" : ("google/protobuf/empty.proto", []),
+    "field_mask" : ("google/protobuf/field_mask.proto", []),
+    "source_context" : ("google/protobuf/source_context.proto", []),
+    "struct" : ("google/protobuf/struct.proto", []),
+    "timestamp" : ("google/protobuf/timestamp.proto", []),
+    "type" : ("google/protobuf/type.proto", ["any", "source_context"]),
+    "wrappers" : ("google/protobuf/wrappers.proto", []),
+}
+
+RELATIVE_WELL_KNOWN_PROTOS = [proto[1][0] for proto in WELL_KNOWN_PROTO_MAP.items()]
 
 genrule(
     name = "link_proto_files",
-    outs = PROTO_FILES,
+    outs = RELATIVE_WELL_KNOWN_PROTOS,
     cmd = """
       for i in $(OUTS); do
         f=$${i#$(@D)/}
         mkdir -p $(@D)/$${f%/*}
-        ln -sf $(INCLUDEDIR)/$$f $(@D)/$$f
+        ln -sf $(PROTOBUF_INCLUDE_PATH)/$$f $(@D)/$$f
       done
     """,
 )
@@ -85,74 +89,9 @@ py_library(
     visibility = ["//visibility:public"],
 )
 
-proto_library(
-    name = "any_proto",
-    srcs = ["google/protobuf/any.proto"],
+[proto_library(
+    name = proto[0] + "_proto",
+    srcs = [proto[1][0]],
+    deps = [dep + "_proto" for dep in proto[1][1]],
     visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "api_proto",
-    srcs = ["google/protobuf/api.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "compiler_plugin_proto",
-    srcs = ["google/protobuf/compiler/plugin.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "descriptor_proto",
-    srcs = ["google/protobuf/descriptor.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "duration_proto",
-    srcs = ["google/protobuf/duration.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "empty_proto",
-    srcs = ["google/protobuf/empty.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "field_mask_proto",
-    srcs = ["google/protobuf/field_mask.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "source_context_proto",
-    srcs = ["google/protobuf/source_context.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "struct_proto",
-    srcs = ["google/protobuf/struct.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "timestamp_proto",
-    srcs = ["google/protobuf/timestamp.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "type_proto",
-    srcs = ["google/protobuf/type.proto"],
-    visibility = ["//visibility:public"],
-)
-
-proto_library(
-    name = "wrappers_proto",
-    srcs = ["google/protobuf/wrappers.proto"],
-    visibility = ["//visibility:public"],
-)
+    ) for proto in WELL_KNOWN_PROTO_MAP.items()]

third_party/systemlibs/syslibs_configure.bzl

@@ -20,6 +20,7 @@ VALID_LIBS = [
     "com_googlesource_code_re2",
     "curl",
     "cython",
+    "dill_archive",
     "double_conversion",
     "enum34_archive",
     "flatbuffers",
@@ -41,7 +42,9 @@ VALID_LIBS = [
     "pybind11",
     "six_archive",
     "snappy",
+    "tblib_archive",
     "termcolor_archive",
+    "typing_extensions_archive",
     "wrapt",
     "zlib",
 ]

third_party/systemlibs/tblib.BUILD

@@ -0,0 +1,12 @@
+licenses(["notice"])  # BSD 3-clause
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "tblib",
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)

third_party/systemlibs/typing_extensions.BUILD

@@ -0,0 +1,16 @@
+# Description:
+#   Backports for the typing module to older Python versions. See
+#   https://github.com/python/typing/blob/master/typing_extensions/README.rst
+
+licenses(["notice"])  # PSF
+
+py_library(
+    name = "typing_extensions",
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+)