From db908716a4c4edee8fe4297fb58423ff29c54f78 Mon Sep 17 00:00:00 2001 From: Jason Zaman Date: Mon, 11 May 2020 10:57:29 +0800 Subject: [PATCH] systemlibs: Update to build against system GRPC - Add libgpr Newer grpc-1.28 has a libgpr.so that is also needed during link time so add it to the linkopts - Add starlark files Several starlark files are load()'d from the GRPC repo, vendor them or add stubs as appropriate when using the system version of grpc. - grpc WORKSPACE deps Several deps were loaded in WORKSPACE that were needed by grpc, they are not needed when building against the system but are difficult to stub out causing the build to fail. grpc_extra_deps.bzl is provided to load all the requirements, so use that from WORKSPACE instead of directly loading each individually. This is also more maintainable going forward since there is less to keep in sync in TF's WORKSPACE file. Signed-off-by: Jason Zaman --- WORKSPACE | 20 +- tensorflow/workspace.bzl | 4 + third_party/systemlibs/grpc.BUILD | 30 ++- .../systemlibs/grpc.bazel.cc_grpc_library.bzl | 105 ++++++++ .../systemlibs/grpc.bazel.generate_cc.bzl | 186 +++++++++++++ .../systemlibs/grpc.bazel.grpc_extra_deps.bzl | 4 + .../systemlibs/grpc.bazel.protobuf.bzl | 244 ++++++++++++++++++ 7 files changed, 571 insertions(+), 22 deletions(-) create mode 100644 third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl create mode 100644 third_party/systemlibs/grpc.bazel.generate_cc.bzl create mode 100644 third_party/systemlibs/grpc.bazel.grpc_extra_deps.bzl create mode 100644 third_party/systemlibs/grpc.bazel.protobuf.bzl diff --git a/WORKSPACE b/WORKSPACE index fa39cedae9b..9db1d9b80eb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -113,26 +113,10 @@ http_archive( # Required for dependency @com_github_grpc_grpc load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") - grpc_deps() -load( - "@build_bazel_rules_apple//apple:repositories.bzl", - "apple_rules_dependencies", -) - -apple_rules_dependencies() - -load( - "@build_bazel_apple_support//lib:repositories.bzl", - "apple_support_dependencies", -) - -apple_support_dependencies() - -load("@upb//bazel:repository_defs.bzl", "bazel_version_repository") - -bazel_version_repository(name = "bazel_version") +load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") +grpc_extra_deps() load("//third_party/googleapis:repository_rules.bzl", "config_googleapis") diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 41c4c836185..a6ea0094dde 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -658,6 +658,10 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): "//third_party/systemlibs:BUILD": "bazel/BUILD", "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", }, urls = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz", diff --git a/third_party/systemlibs/grpc.BUILD b/third_party/systemlibs/grpc.BUILD index fd90eb0dd3d..8b703f11556 100644 --- a/third_party/systemlibs/grpc.BUILD +++ b/third_party/systemlibs/grpc.BUILD @@ -7,25 +7,47 @@ filegroup( cc_library( name = "grpc", - linkopts = ["-lgrpc"], + linkopts = [ + "-lgrpc", + "-lgpr", + ], visibility = ["//visibility:public"], ) cc_library( name = "grpc++", - linkopts = ["-lgrpc++"], + linkopts = [ + "-lgrpc++", + "-lgpr", + ], + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc++_public_hdrs", + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc++_codegen_proto", visibility = ["//visibility:public"], ) cc_library( name = "grpc_unsecure", - linkopts = ["-lgrpc_unsecure"], + linkopts = [ + "-lgrpc_unsecure", + "-lgpr", + ], visibility = ["//visibility:public"], ) cc_library( name = "grpc++_unsecure", - linkopts = ["-lgrpc++_unsecure"], + linkopts = [ + "-lgrpc++_unsecure", + "-lgpr", + ], visibility = ["//visibility:public"], ) diff --git a/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl b/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl new file mode 100644 index 00000000000..e427328c39b --- /dev/null +++ b/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl @@ -0,0 +1,105 @@ +"""Generates and compiles C++ grpc stubs from proto_library rules.""" + +load("@com_github_grpc_grpc//bazel:generate_cc.bzl", "generate_cc") +load("@com_github_grpc_grpc//bazel:protobuf.bzl", "well_known_proto_libs") + +def cc_grpc_library( + name, + srcs, + deps, + proto_only = False, + well_known_protos = False, + generate_mocks = False, + use_external = False, + grpc_only = False, + **kwargs): + """Generates C++ grpc classes for services defined in a proto file. + + If grpc_only is True, this rule is compatible with proto_library and + cc_proto_library native rules such that it expects proto_library target + as srcs argument and generates only grpc library classes, expecting + protobuf messages classes library (cc_proto_library target) to be passed in + deps argument. By default grpc_only is False which makes this rule to behave + in a backwards-compatible mode (trying to generate both proto and grpc + classes). + + Assumes the generated classes will be used in cc_api_version = 2. + + Args: + name (str): Name of rule. + srcs (list): A single .proto file which contains services definitions, + or if grpc_only parameter is True, a single proto_library which + contains services descriptors. + deps (list): A list of C++ proto_library (or cc_proto_library) which + provides the compiled code of any message that the services depend on. + proto_only (bool): If True, create only C++ proto classes library, + avoid creating C++ grpc classes library (expect it in deps). + Deprecated, use native cc_proto_library instead. False by default. + well_known_protos (bool): Should this library additionally depend on + well known protos. Deprecated, the well known protos should be + specified as explicit dependencies of the proto_library target + (passed in srcs parameter) instead. False by default. + generate_mocks (bool): when True, Google Mock code for client stub is + generated. False by default. + use_external (bool): Not used. + grpc_only (bool): if True, generate only grpc library, expecting + protobuf messages library (cc_proto_library target) to be passed as + deps. False by default (will become True by default eventually). + **kwargs: rest of arguments, e.g., compatible_with and visibility + """ + if len(srcs) > 1: + fail("Only one srcs value supported", "srcs") + if grpc_only and proto_only: + fail("A mutualy exclusive configuration is specified: grpc_only = True and proto_only = True") + + extra_deps = [] + proto_targets = [] + + if not grpc_only: + proto_target = "_" + name + "_only" + cc_proto_target = name if proto_only else "_" + name + "_cc_proto" + + proto_deps = ["_" + dep + "_only" for dep in deps if dep.find(":") == -1] + proto_deps += [dep.split(":")[0] + ":" + "_" + dep.split(":")[1] + "_only" for dep in deps if dep.find(":") != -1] + if well_known_protos: + proto_deps += well_known_proto_libs() + + native.proto_library( + name = proto_target, + srcs = srcs, + deps = proto_deps, + **kwargs + ) + + native.cc_proto_library( + name = cc_proto_target, + deps = [":" + proto_target], + **kwargs + ) + extra_deps.append(":" + cc_proto_target) + proto_targets.append(proto_target) + else: + if not srcs: + fail("srcs cannot be empty", "srcs") + proto_targets += srcs + + if not proto_only: + codegen_grpc_target = "_" + name + "_grpc_codegen" + generate_cc( + name = codegen_grpc_target, + srcs = proto_targets, + plugin = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin", + well_known_protos = well_known_protos, + generate_mocks = generate_mocks, + **kwargs + ) + + native.cc_library( + name = name, + srcs = [":" + codegen_grpc_target], + hdrs = [":" + codegen_grpc_target], + deps = deps + + extra_deps + + ["@com_github_grpc_grpc//:grpc++_codegen_proto"], + **kwargs + ) diff --git a/third_party/systemlibs/grpc.bazel.generate_cc.bzl b/third_party/systemlibs/grpc.bazel.generate_cc.bzl new file mode 100644 index 00000000000..1534e526fd1 --- /dev/null +++ b/third_party/systemlibs/grpc.bazel.generate_cc.bzl @@ -0,0 +1,186 @@ +"""Generates C++ grpc stubs from proto_library rules. + +This is an internal rule used by cc_grpc_library, and shouldn't be used +directly. +""" + +load( + "@com_github_grpc_grpc//bazel:protobuf.bzl", + "get_include_directory", + "get_plugin_args", + "get_proto_root", + "proto_path_to_generated_filename", +) + +_GRPC_PROTO_HEADER_FMT = "{}.grpc.pb.h" +_GRPC_PROTO_SRC_FMT = "{}.grpc.pb.cc" +_GRPC_PROTO_MOCK_HEADER_FMT = "{}_mock.grpc.pb.h" +_PROTO_HEADER_FMT = "{}.pb.h" +_PROTO_SRC_FMT = "{}.pb.cc" + +def _strip_package_from_path(label_package, file): + prefix_len = 0 + if not file.is_source and file.path.startswith(file.root.path): + prefix_len = len(file.root.path) + 1 + + path = file.path + if len(label_package) == 0: + return path + if not path.startswith(label_package + "/", prefix_len): + fail("'{}' does not lie within '{}'.".format(path, label_package)) + return path[prefix_len + len(label_package + "/"):] + +def _get_srcs_file_path(file): + if not file.is_source and file.path.startswith(file.root.path): + return file.path[len(file.root.path) + 1:] + return file.path + +def _join_directories(directories): + massaged_directories = [directory for directory in directories if len(directory) != 0] + return "/".join(massaged_directories) + +def generate_cc_impl(ctx): + """Implementation of the generate_cc rule.""" + protos = [f for src in ctx.attr.srcs for f in src[ProtoInfo].check_deps_sources.to_list()] + includes = [ + f + for src in ctx.attr.srcs + for f in src[ProtoInfo].transitive_imports.to_list() + ] + outs = [] + proto_root = get_proto_root( + ctx.label.workspace_root, + ) + + label_package = _join_directories([ctx.label.workspace_root, ctx.label.package]) + if ctx.executable.plugin: + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _GRPC_PROTO_HEADER_FMT, + ) + for proto in protos + ] + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _GRPC_PROTO_SRC_FMT, + ) + for proto in protos + ] + if ctx.attr.generate_mocks: + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _GRPC_PROTO_MOCK_HEADER_FMT, + ) + for proto in protos + ] + else: + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _PROTO_HEADER_FMT, + ) + for proto in protos + ] + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _PROTO_SRC_FMT, + ) + for proto in protos + ] + out_files = [ctx.actions.declare_file(out) for out in outs] + dir_out = str(ctx.genfiles_dir.path + proto_root) + + arguments = [] + if ctx.executable.plugin: + arguments += get_plugin_args( + ctx.executable.plugin, + ctx.attr.flags, + dir_out, + ctx.attr.generate_mocks, + ) + tools = [ctx.executable.plugin] + else: + arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out] + tools = [] + + arguments += [ + "--proto_path={}".format(get_include_directory(i)) + for i in includes + ] + + # Include the output directory so that protoc puts the generated code in the + # right directory. + arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)] + arguments += [_get_srcs_file_path(proto) for proto in protos] + + # create a list of well known proto files if the argument is non-None + well_known_proto_files = [] + if ctx.attr.well_known_protos: + f = ctx.attr.well_known_protos.files.to_list()[0].dirname + if f != "external/com_google_protobuf/src/google/protobuf": + print( + "Error: Only @com_google_protobuf//:well_known_protos is supported", + ) + else: + # f points to "external/com_google_protobuf/src/google/protobuf" + # add -I argument to protoc so it knows where to look for the proto files. + arguments += ["-I{0}".format(f + "/../..")] + well_known_proto_files = [ + f + for f in ctx.attr.well_known_protos.files.to_list() + ] + + ctx.actions.run( + inputs = protos + includes + well_known_proto_files, + tools = tools, + outputs = out_files, + executable = ctx.executable._protoc, + arguments = arguments, + ) + + return struct(files = depset(out_files)) + +_generate_cc = rule( + attrs = { + "srcs": attr.label_list( + mandatory = True, + allow_empty = False, + providers = [ProtoInfo], + ), + "plugin": attr.label( + executable = True, + providers = ["files_to_run"], + cfg = "host", + ), + "flags": attr.string_list( + mandatory = False, + allow_empty = True, + ), + "well_known_protos": attr.label(mandatory = False), + "generate_mocks": attr.bool( + default = False, + mandatory = False, + ), + "_protoc": attr.label( + default = Label("@com_google_protobuf//:protoc"), + executable = True, + cfg = "host", + ), + }, + # We generate .h files, so we need to output to genfiles. + output_to_genfiles = True, + implementation = generate_cc_impl, +) + +def generate_cc(well_known_protos, **kwargs): + if well_known_protos: + _generate_cc( + well_known_protos = "@com_google_protobuf//:well_known_protos", + **kwargs + ) + else: + _generate_cc(**kwargs) diff --git a/third_party/systemlibs/grpc.bazel.grpc_extra_deps.bzl b/third_party/systemlibs/grpc.bazel.grpc_extra_deps.bzl new file mode 100644 index 00000000000..631c93af047 --- /dev/null +++ b/third_party/systemlibs/grpc.bazel.grpc_extra_deps.bzl @@ -0,0 +1,4 @@ +"""Stub version of @com_github_grpc_grpc//bazel:grpc_extra_deps.bzl necessary for TF system libs""" + +def grpc_extra_deps(): + pass diff --git a/third_party/systemlibs/grpc.bazel.protobuf.bzl b/third_party/systemlibs/grpc.bazel.protobuf.bzl new file mode 100644 index 00000000000..a59b2bd2155 --- /dev/null +++ b/third_party/systemlibs/grpc.bazel.protobuf.bzl @@ -0,0 +1,244 @@ +"""Utility functions for generating protobuf code.""" + +_PROTO_EXTENSION = ".proto" +_VIRTUAL_IMPORTS = "/_virtual_imports/" + +def well_known_proto_libs(): + return [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:api_proto", + "@com_google_protobuf//:compiler_plugin_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:field_mask_proto", + "@com_google_protobuf//:source_context_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:type_proto", + "@com_google_protobuf//:wrappers_proto", + ] + +def get_proto_root(workspace_root): + """Gets the root protobuf directory. + + Args: + workspace_root: context.label.workspace_root + + Returns: + The directory relative to which generated include paths should be. + """ + if workspace_root: + return "/{}".format(workspace_root) + else: + return "" + +def _strip_proto_extension(proto_filename): + if not proto_filename.endswith(_PROTO_EXTENSION): + fail('"{}" does not end with "{}"'.format( + proto_filename, + _PROTO_EXTENSION, + )) + return proto_filename[:-len(_PROTO_EXTENSION)] + +def proto_path_to_generated_filename(proto_path, fmt_str): + """Calculates the name of a generated file for a protobuf path. + + For example, "examples/protos/helloworld.proto" might map to + "helloworld.pb.h". + + Args: + proto_path: The path to the .proto file. + fmt_str: A format string used to calculate the generated filename. For + example, "{}.pb.h" might be used to calculate a C++ header filename. + + Returns: + The generated filename. + """ + return fmt_str.format(_strip_proto_extension(proto_path)) + +def get_include_directory(source_file): + """Returns the include directory path for the source_file. + + I.e. all of the include statements within the given source_file + are calculated relative to the directory returned by this method. + + The returned directory path can be used as the "--proto_path=" argument + value. + + Args: + source_file: A proto file. + + Returns: + The include directory path for the source_file. + """ + directory = source_file.path + prefix_len = 0 + + if is_in_virtual_imports(source_file): + root, relative = source_file.path.split(_VIRTUAL_IMPORTS, 2) + result = root + _VIRTUAL_IMPORTS + relative.split("/", 1)[0] + return result + + if not source_file.is_source and directory.startswith(source_file.root.path): + prefix_len = len(source_file.root.path) + 1 + + if directory.startswith("external", prefix_len): + external_separator = directory.find("/", prefix_len) + repository_separator = directory.find("/", external_separator + 1) + return directory[:repository_separator] + else: + return source_file.root.path if source_file.root.path else "." + +def get_plugin_args( + plugin, + flags, + dir_out, + generate_mocks, + plugin_name = "PLUGIN"): + """Returns arguments configuring protoc to use a plugin for a language. + + Args: + plugin: An executable file to run as the protoc plugin. + flags: The plugin flags to be passed to protoc. + dir_out: The output directory for the plugin. + generate_mocks: A bool indicating whether to generate mocks. + plugin_name: A name of the plugin, it is required to be unique when there + are more than one plugin used in a single protoc command. + Returns: + A list of protoc arguments configuring the plugin. + """ + augmented_flags = list(flags) + if generate_mocks: + augmented_flags.append("generate_mock_code=true") + + augmented_dir_out = dir_out + if augmented_flags: + augmented_dir_out = ",".join(augmented_flags) + ":" + dir_out + + return [ + "--plugin=protoc-gen-{plugin_name}={plugin_path}".format( + plugin_name = plugin_name, + plugin_path = plugin.path, + ), + "--{plugin_name}_out={dir_out}".format( + plugin_name = plugin_name, + dir_out = augmented_dir_out, + ), + ] + +def _get_staged_proto_file(context, source_file): + if source_file.dirname == context.label.package or \ + is_in_virtual_imports(source_file): + return source_file + else: + copied_proto = context.actions.declare_file(source_file.basename) + context.actions.run_shell( + inputs = [source_file], + outputs = [copied_proto], + command = "cp {} {}".format(source_file.path, copied_proto.path), + mnemonic = "CopySourceProto", + ) + return copied_proto + +def protos_from_context(context): + """Copies proto files to the appropriate location. + + Args: + context: The ctx object for the rule. + + Returns: + A list of the protos. + """ + protos = [] + for src in context.attr.deps: + for file in src[ProtoInfo].direct_sources: + protos.append(_get_staged_proto_file(context, file)) + return protos + +def includes_from_deps(deps): + """Get includes from rule dependencies.""" + return [ + file + for src in deps + for file in src[ProtoInfo].transitive_imports.to_list() + ] + +def get_proto_arguments(protos, genfiles_dir_path): + """Get the protoc arguments specifying which protos to compile.""" + arguments = [] + for proto in protos: + strip_prefix_len = 0 + if is_in_virtual_imports(proto): + incl_directory = get_include_directory(proto) + if proto.path.startswith(incl_directory): + strip_prefix_len = len(incl_directory) + 1 + elif proto.path.startswith(genfiles_dir_path): + strip_prefix_len = len(genfiles_dir_path) + 1 + + arguments.append(proto.path[strip_prefix_len:]) + + return arguments + +def declare_out_files(protos, context, generated_file_format): + """Declares and returns the files to be generated.""" + + out_file_paths = [] + for proto in protos: + if not is_in_virtual_imports(proto): + out_file_paths.append(proto.basename) + else: + path = proto.path[proto.path.index(_VIRTUAL_IMPORTS) + 1:] + out_file_paths.append(path) + + return [ + context.actions.declare_file( + proto_path_to_generated_filename( + out_file_path, + generated_file_format, + ), + ) + for out_file_path in out_file_paths + ] + +def get_out_dir(protos, context): + """ Returns the calculated value for --_out= protoc argument based on + the input source proto files and current context. + + Args: + protos: A list of protos to be used as source files in protoc command + context: A ctx object for the rule. + Returns: + The value of --_out= argument. + """ + at_least_one_virtual = 0 + for proto in protos: + if is_in_virtual_imports(proto): + at_least_one_virtual = True + elif at_least_one_virtual: + fail("Proto sources must be either all virtual imports or all real") + if at_least_one_virtual: + out_dir = get_include_directory(protos[0]) + ws_root = protos[0].owner.workspace_root + if ws_root and out_dir.find(ws_root) >= 0: + out_dir = "".join(out_dir.rsplit(ws_root, 1)) + return struct( + path = out_dir, + import_path = out_dir[out_dir.find(_VIRTUAL_IMPORTS) + 1:], + ) + return struct(path = context.genfiles_dir.path, import_path = None) + +def is_in_virtual_imports(source_file, virtual_folder = _VIRTUAL_IMPORTS): + """Determines if source_file is virtual (is placed in _virtual_imports + subdirectory). The output of all proto_library targets which use + import_prefix and/or strip_import_prefix arguments is placed under + _virtual_imports directory. + + Args: + source_file: A proto file. + virtual_folder: The virtual folder name (is set to "_virtual_imports" + by default) + Returns: + True if source_file is located under _virtual_imports, False otherwise. + """ + return not source_file.is_source and virtual_folder in source_file.path