diff --git a/WORKSPACE b/WORKSPACE index fa39cedae9b..9db1d9b80eb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -113,26 +113,10 @@ http_archive( # Required for dependency @com_github_grpc_grpc load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") - grpc_deps() -load( - "@build_bazel_rules_apple//apple:repositories.bzl", - "apple_rules_dependencies", -) - -apple_rules_dependencies() - -load( - "@build_bazel_apple_support//lib:repositories.bzl", - "apple_support_dependencies", -) - -apple_support_dependencies() - -load("@upb//bazel:repository_defs.bzl", "bazel_version_repository") - -bazel_version_repository(name = "bazel_version") +load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") +grpc_extra_deps() load("//third_party/googleapis:repository_rules.bzl", "config_googleapis") diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 41c4c836185..a6ea0094dde 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -658,6 +658,10 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): "//third_party/systemlibs:BUILD": "bazel/BUILD", "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", }, urls = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz", diff --git a/third_party/systemlibs/grpc.BUILD b/third_party/systemlibs/grpc.BUILD index fd90eb0dd3d..8b703f11556 100644 --- a/third_party/systemlibs/grpc.BUILD +++ b/third_party/systemlibs/grpc.BUILD @@ -7,25 +7,47 @@ filegroup( cc_library( name = "grpc", - linkopts = ["-lgrpc"], + linkopts = [ + "-lgrpc", + "-lgpr", + ], visibility = ["//visibility:public"], ) cc_library( name = "grpc++", - linkopts = ["-lgrpc++"], + linkopts = [ + "-lgrpc++", + "-lgpr", + ], + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc++_public_hdrs", + visibility = ["//visibility:public"], +) + +cc_library( + name = "grpc++_codegen_proto", visibility = ["//visibility:public"], ) cc_library( name = "grpc_unsecure", - linkopts = ["-lgrpc_unsecure"], + linkopts = [ + "-lgrpc_unsecure", + "-lgpr", + ], visibility = ["//visibility:public"], ) cc_library( name = "grpc++_unsecure", - linkopts = ["-lgrpc++_unsecure"], + linkopts = [ + "-lgrpc++_unsecure", + "-lgpr", + ], visibility = ["//visibility:public"], ) diff --git a/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl b/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl new file mode 100644 index 00000000000..e427328c39b --- /dev/null +++ b/third_party/systemlibs/grpc.bazel.cc_grpc_library.bzl @@ -0,0 +1,105 @@ +"""Generates and compiles C++ grpc stubs from proto_library rules.""" + +load("@com_github_grpc_grpc//bazel:generate_cc.bzl", "generate_cc") +load("@com_github_grpc_grpc//bazel:protobuf.bzl", "well_known_proto_libs") + +def cc_grpc_library( + name, + srcs, + deps, + proto_only = False, + well_known_protos = False, + generate_mocks = False, + use_external = False, + grpc_only = False, + **kwargs): + """Generates C++ grpc classes for services defined in a proto file. + + If grpc_only is True, this rule is compatible with proto_library and + cc_proto_library native rules such that it expects proto_library target + as srcs argument and generates only grpc library classes, expecting + protobuf messages classes library (cc_proto_library target) to be passed in + deps argument. By default grpc_only is False which makes this rule to behave + in a backwards-compatible mode (trying to generate both proto and grpc + classes). + + Assumes the generated classes will be used in cc_api_version = 2. + + Args: + name (str): Name of rule. + srcs (list): A single .proto file which contains services definitions, + or if grpc_only parameter is True, a single proto_library which + contains services descriptors. + deps (list): A list of C++ proto_library (or cc_proto_library) which + provides the compiled code of any message that the services depend on. + proto_only (bool): If True, create only C++ proto classes library, + avoid creating C++ grpc classes library (expect it in deps). + Deprecated, use native cc_proto_library instead. False by default. + well_known_protos (bool): Should this library additionally depend on + well known protos. Deprecated, the well known protos should be + specified as explicit dependencies of the proto_library target + (passed in srcs parameter) instead. False by default. + generate_mocks (bool): when True, Google Mock code for client stub is + generated. False by default. + use_external (bool): Not used. + grpc_only (bool): if True, generate only grpc library, expecting + protobuf messages library (cc_proto_library target) to be passed as + deps. False by default (will become True by default eventually). + **kwargs: rest of arguments, e.g., compatible_with and visibility + """ + if len(srcs) > 1: + fail("Only one srcs value supported", "srcs") + if grpc_only and proto_only: + fail("A mutualy exclusive configuration is specified: grpc_only = True and proto_only = True") + + extra_deps = [] + proto_targets = [] + + if not grpc_only: + proto_target = "_" + name + "_only" + cc_proto_target = name if proto_only else "_" + name + "_cc_proto" + + proto_deps = ["_" + dep + "_only" for dep in deps if dep.find(":") == -1] + proto_deps += [dep.split(":")[0] + ":" + "_" + dep.split(":")[1] + "_only" for dep in deps if dep.find(":") != -1] + if well_known_protos: + proto_deps += well_known_proto_libs() + + native.proto_library( + name = proto_target, + srcs = srcs, + deps = proto_deps, + **kwargs + ) + + native.cc_proto_library( + name = cc_proto_target, + deps = [":" + proto_target], + **kwargs + ) + extra_deps.append(":" + cc_proto_target) + proto_targets.append(proto_target) + else: + if not srcs: + fail("srcs cannot be empty", "srcs") + proto_targets += srcs + + if not proto_only: + codegen_grpc_target = "_" + name + "_grpc_codegen" + generate_cc( + name = codegen_grpc_target, + srcs = proto_targets, + plugin = "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin", + well_known_protos = well_known_protos, + generate_mocks = generate_mocks, + **kwargs + ) + + native.cc_library( + name = name, + srcs = [":" + codegen_grpc_target], + hdrs = [":" + codegen_grpc_target], + deps = deps + + extra_deps + + ["@com_github_grpc_grpc//:grpc++_codegen_proto"], + **kwargs + ) diff --git a/third_party/systemlibs/grpc.bazel.generate_cc.bzl b/third_party/systemlibs/grpc.bazel.generate_cc.bzl new file mode 100644 index 00000000000..1534e526fd1 --- /dev/null +++ b/third_party/systemlibs/grpc.bazel.generate_cc.bzl @@ -0,0 +1,186 @@ +"""Generates C++ grpc stubs from proto_library rules. + +This is an internal rule used by cc_grpc_library, and shouldn't be used +directly. +""" + +load( + "@com_github_grpc_grpc//bazel:protobuf.bzl", + "get_include_directory", + "get_plugin_args", + "get_proto_root", + "proto_path_to_generated_filename", +) + +_GRPC_PROTO_HEADER_FMT = "{}.grpc.pb.h" +_GRPC_PROTO_SRC_FMT = "{}.grpc.pb.cc" +_GRPC_PROTO_MOCK_HEADER_FMT = "{}_mock.grpc.pb.h" +_PROTO_HEADER_FMT = "{}.pb.h" +_PROTO_SRC_FMT = "{}.pb.cc" + +def _strip_package_from_path(label_package, file): + prefix_len = 0 + if not file.is_source and file.path.startswith(file.root.path): + prefix_len = len(file.root.path) + 1 + + path = file.path + if len(label_package) == 0: + return path + if not path.startswith(label_package + "/", prefix_len): + fail("'{}' does not lie within '{}'.".format(path, label_package)) + return path[prefix_len + len(label_package + "/"):] + +def _get_srcs_file_path(file): + if not file.is_source and file.path.startswith(file.root.path): + return file.path[len(file.root.path) + 1:] + return file.path + +def _join_directories(directories): + massaged_directories = [directory for directory in directories if len(directory) != 0] + return "/".join(massaged_directories) + +def generate_cc_impl(ctx): + """Implementation of the generate_cc rule.""" + protos = [f for src in ctx.attr.srcs for f in src[ProtoInfo].check_deps_sources.to_list()] + includes = [ + f + for src in ctx.attr.srcs + for f in src[ProtoInfo].transitive_imports.to_list() + ] + outs = [] + proto_root = get_proto_root( + ctx.label.workspace_root, + ) + + label_package = _join_directories([ctx.label.workspace_root, ctx.label.package]) + if ctx.executable.plugin: + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _GRPC_PROTO_HEADER_FMT, + ) + for proto in protos + ] + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _GRPC_PROTO_SRC_FMT, + ) + for proto in protos + ] + if ctx.attr.generate_mocks: + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _GRPC_PROTO_MOCK_HEADER_FMT, + ) + for proto in protos + ] + else: + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _PROTO_HEADER_FMT, + ) + for proto in protos + ] + outs += [ + proto_path_to_generated_filename( + _strip_package_from_path(label_package, proto), + _PROTO_SRC_FMT, + ) + for proto in protos + ] + out_files = [ctx.actions.declare_file(out) for out in outs] + dir_out = str(ctx.genfiles_dir.path + proto_root) + + arguments = [] + if ctx.executable.plugin: + arguments += get_plugin_args( + ctx.executable.plugin, + ctx.attr.flags, + dir_out, + ctx.attr.generate_mocks, + ) + tools = [ctx.executable.plugin] + else: + arguments += ["--cpp_out=" + ",".join(ctx.attr.flags) + ":" + dir_out] + tools = [] + + arguments += [ + "--proto_path={}".format(get_include_directory(i)) + for i in includes + ] + + # Include the output directory so that protoc puts the generated code in the + # right directory. + arguments += ["--proto_path={0}{1}".format(dir_out, proto_root)] + arguments += [_get_srcs_file_path(proto) for proto in protos] + + # create a list of well known proto files if the argument is non-None + well_known_proto_files = [] + if ctx.attr.well_known_protos: + f = ctx.attr.well_known_protos.files.to_list()[0].dirname + if f != "external/com_google_protobuf/src/google/protobuf": + print( + "Error: Only @com_google_protobuf//:well_known_protos is supported", + ) + else: + # f points to "external/com_google_protobuf/src/google/protobuf" + # add -I argument to protoc so it knows where to look for the proto files. + arguments += ["-I{0}".format(f + "/../..")] + well_known_proto_files = [ + f + for f in ctx.attr.well_known_protos.files.to_list() + ] + + ctx.actions.run( + inputs = protos + includes + well_known_proto_files, + tools = tools, + outputs = out_files, + executable = ctx.executable._protoc, + arguments = arguments, + ) + + return struct(files = depset(out_files)) + +_generate_cc = rule( + attrs = { + "srcs": attr.label_list( + mandatory = True, + allow_empty = False, + providers = [ProtoInfo], + ), + "plugin": attr.label( + executable = True, + providers = ["files_to_run"], + cfg = "host", + ), + "flags": attr.string_list( + mandatory = False, + allow_empty = True, + ), + "well_known_protos": attr.label(mandatory = False), + "generate_mocks": attr.bool( + default = False, + mandatory = False, + ), + "_protoc": attr.label( + default = Label("@com_google_protobuf//:protoc"), + executable = True, + cfg = "host", + ), + }, + # We generate .h files, so we need to output to genfiles. + output_to_genfiles = True, + implementation = generate_cc_impl, +) + +def generate_cc(well_known_protos, **kwargs): + if well_known_protos: + _generate_cc( + well_known_protos = "@com_google_protobuf//:well_known_protos", + **kwargs + ) + else: + _generate_cc(**kwargs) diff --git a/third_party/systemlibs/grpc.bazel.grpc_extra_deps.bzl b/third_party/systemlibs/grpc.bazel.grpc_extra_deps.bzl new file mode 100644 index 00000000000..631c93af047 --- /dev/null +++ b/third_party/systemlibs/grpc.bazel.grpc_extra_deps.bzl @@ -0,0 +1,4 @@ +"""Stub version of @com_github_grpc_grpc//bazel:grpc_extra_deps.bzl necessary for TF system libs""" + +def grpc_extra_deps(): + pass diff --git a/third_party/systemlibs/grpc.bazel.protobuf.bzl b/third_party/systemlibs/grpc.bazel.protobuf.bzl new file mode 100644 index 00000000000..a59b2bd2155 --- /dev/null +++ b/third_party/systemlibs/grpc.bazel.protobuf.bzl @@ -0,0 +1,244 @@ +"""Utility functions for generating protobuf code.""" + +_PROTO_EXTENSION = ".proto" +_VIRTUAL_IMPORTS = "/_virtual_imports/" + +def well_known_proto_libs(): + return [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:api_proto", + "@com_google_protobuf//:compiler_plugin_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:field_mask_proto", + "@com_google_protobuf//:source_context_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:type_proto", + "@com_google_protobuf//:wrappers_proto", + ] + +def get_proto_root(workspace_root): + """Gets the root protobuf directory. + + Args: + workspace_root: context.label.workspace_root + + Returns: + The directory relative to which generated include paths should be. + """ + if workspace_root: + return "/{}".format(workspace_root) + else: + return "" + +def _strip_proto_extension(proto_filename): + if not proto_filename.endswith(_PROTO_EXTENSION): + fail('"{}" does not end with "{}"'.format( + proto_filename, + _PROTO_EXTENSION, + )) + return proto_filename[:-len(_PROTO_EXTENSION)] + +def proto_path_to_generated_filename(proto_path, fmt_str): + """Calculates the name of a generated file for a protobuf path. + + For example, "examples/protos/helloworld.proto" might map to + "helloworld.pb.h". + + Args: + proto_path: The path to the .proto file. + fmt_str: A format string used to calculate the generated filename. For + example, "{}.pb.h" might be used to calculate a C++ header filename. + + Returns: + The generated filename. + """ + return fmt_str.format(_strip_proto_extension(proto_path)) + +def get_include_directory(source_file): + """Returns the include directory path for the source_file. + + I.e. all of the include statements within the given source_file + are calculated relative to the directory returned by this method. + + The returned directory path can be used as the "--proto_path=" argument + value. + + Args: + source_file: A proto file. + + Returns: + The include directory path for the source_file. + """ + directory = source_file.path + prefix_len = 0 + + if is_in_virtual_imports(source_file): + root, relative = source_file.path.split(_VIRTUAL_IMPORTS, 2) + result = root + _VIRTUAL_IMPORTS + relative.split("/", 1)[0] + return result + + if not source_file.is_source and directory.startswith(source_file.root.path): + prefix_len = len(source_file.root.path) + 1 + + if directory.startswith("external", prefix_len): + external_separator = directory.find("/", prefix_len) + repository_separator = directory.find("/", external_separator + 1) + return directory[:repository_separator] + else: + return source_file.root.path if source_file.root.path else "." + +def get_plugin_args( + plugin, + flags, + dir_out, + generate_mocks, + plugin_name = "PLUGIN"): + """Returns arguments configuring protoc to use a plugin for a language. + + Args: + plugin: An executable file to run as the protoc plugin. + flags: The plugin flags to be passed to protoc. + dir_out: The output directory for the plugin. + generate_mocks: A bool indicating whether to generate mocks. + plugin_name: A name of the plugin, it is required to be unique when there + are more than one plugin used in a single protoc command. + Returns: + A list of protoc arguments configuring the plugin. + """ + augmented_flags = list(flags) + if generate_mocks: + augmented_flags.append("generate_mock_code=true") + + augmented_dir_out = dir_out + if augmented_flags: + augmented_dir_out = ",".join(augmented_flags) + ":" + dir_out + + return [ + "--plugin=protoc-gen-{plugin_name}={plugin_path}".format( + plugin_name = plugin_name, + plugin_path = plugin.path, + ), + "--{plugin_name}_out={dir_out}".format( + plugin_name = plugin_name, + dir_out = augmented_dir_out, + ), + ] + +def _get_staged_proto_file(context, source_file): + if source_file.dirname == context.label.package or \ + is_in_virtual_imports(source_file): + return source_file + else: + copied_proto = context.actions.declare_file(source_file.basename) + context.actions.run_shell( + inputs = [source_file], + outputs = [copied_proto], + command = "cp {} {}".format(source_file.path, copied_proto.path), + mnemonic = "CopySourceProto", + ) + return copied_proto + +def protos_from_context(context): + """Copies proto files to the appropriate location. + + Args: + context: The ctx object for the rule. + + Returns: + A list of the protos. + """ + protos = [] + for src in context.attr.deps: + for file in src[ProtoInfo].direct_sources: + protos.append(_get_staged_proto_file(context, file)) + return protos + +def includes_from_deps(deps): + """Get includes from rule dependencies.""" + return [ + file + for src in deps + for file in src[ProtoInfo].transitive_imports.to_list() + ] + +def get_proto_arguments(protos, genfiles_dir_path): + """Get the protoc arguments specifying which protos to compile.""" + arguments = [] + for proto in protos: + strip_prefix_len = 0 + if is_in_virtual_imports(proto): + incl_directory = get_include_directory(proto) + if proto.path.startswith(incl_directory): + strip_prefix_len = len(incl_directory) + 1 + elif proto.path.startswith(genfiles_dir_path): + strip_prefix_len = len(genfiles_dir_path) + 1 + + arguments.append(proto.path[strip_prefix_len:]) + + return arguments + +def declare_out_files(protos, context, generated_file_format): + """Declares and returns the files to be generated.""" + + out_file_paths = [] + for proto in protos: + if not is_in_virtual_imports(proto): + out_file_paths.append(proto.basename) + else: + path = proto.path[proto.path.index(_VIRTUAL_IMPORTS) + 1:] + out_file_paths.append(path) + + return [ + context.actions.declare_file( + proto_path_to_generated_filename( + out_file_path, + generated_file_format, + ), + ) + for out_file_path in out_file_paths + ] + +def get_out_dir(protos, context): + """ Returns the calculated value for --_out= protoc argument based on + the input source proto files and current context. + + Args: + protos: A list of protos to be used as source files in protoc command + context: A ctx object for the rule. + Returns: + The value of --_out= argument. + """ + at_least_one_virtual = 0 + for proto in protos: + if is_in_virtual_imports(proto): + at_least_one_virtual = True + elif at_least_one_virtual: + fail("Proto sources must be either all virtual imports or all real") + if at_least_one_virtual: + out_dir = get_include_directory(protos[0]) + ws_root = protos[0].owner.workspace_root + if ws_root and out_dir.find(ws_root) >= 0: + out_dir = "".join(out_dir.rsplit(ws_root, 1)) + return struct( + path = out_dir, + import_path = out_dir[out_dir.find(_VIRTUAL_IMPORTS) + 1:], + ) + return struct(path = context.genfiles_dir.path, import_path = None) + +def is_in_virtual_imports(source_file, virtual_folder = _VIRTUAL_IMPORTS): + """Determines if source_file is virtual (is placed in _virtual_imports + subdirectory). The output of all proto_library targets which use + import_prefix and/or strip_import_prefix arguments is placed under + _virtual_imports directory. + + Args: + source_file: A proto file. + virtual_folder: The virtual folder name (is set to "_virtual_imports" + by default) + Returns: + True if source_file is located under _virtual_imports, False otherwise. + """ + return not source_file.is_source and virtual_folder in source_file.path