Merge pull request #24004 from perfinion:protobuf

PiperOrigin-RevId: 224213775
This commit is contained in:
TensorFlower Gardener 2018-12-05 13:49:04 -08:00
commit ecf243924e
4 changed files with 544 additions and 0 deletions

View File

@ -357,6 +357,10 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
name = "protobuf_archive",
sha256 = PROTOBUF_SHA256,
strip_prefix = PROTOBUF_STRIP_PREFIX,
system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"),
system_link_files = {
"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
},
urls = PROTOBUF_URLS,
)
@ -367,6 +371,10 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
name = "com_google_protobuf",
sha256 = PROTOBUF_SHA256,
strip_prefix = PROTOBUF_STRIP_PREFIX,
system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"),
system_link_files = {
"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
},
urls = PROTOBUF_URLS,
)
@ -374,6 +382,10 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
name = "com_google_protobuf_cc",
sha256 = PROTOBUF_SHA256,
strip_prefix = PROTOBUF_STRIP_PREFIX,
system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"),
system_link_files = {
"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
},
urls = PROTOBUF_URLS,
)

104
third_party/systemlibs/protobuf.BUILD vendored Normal file
View File

@ -0,0 +1,104 @@
load(
"@protobuf_archive//:protobuf.bzl",
"proto_gen",
"py_proto_library",
"cc_proto_library",
)
licenses(["notice"])
filegroup(
name = "LICENSE",
visibility = ["//visibility:public"],
)
HEADERS = [
"google/protobuf/any.pb.h",
"google/protobuf/any.proto",
"google/protobuf/arena.h",
"google/protobuf/compiler/importer.h",
"google/protobuf/descriptor.h",
"google/protobuf/descriptor.pb.h",
"google/protobuf/descriptor.proto",
"google/protobuf/duration.pb.h",
"google/protobuf/duration.proto",
"google/protobuf/dynamic_message.h",
"google/protobuf/empty.pb.h",
"google/protobuf/empty.proto",
"google/protobuf/field_mask.pb.h",
"google/protobuf/field_mask.proto",
"google/protobuf/io/coded_stream.h",
"google/protobuf/io/zero_copy_stream.h",
"google/protobuf/io/zero_copy_stream_impl_lite.h",
"google/protobuf/map.h",
"google/protobuf/repeated_field.h",
"google/protobuf/text_format.h",
"google/protobuf/timestamp.pb.h",
"google/protobuf/timestamp.proto",
"google/protobuf/util/json_util.h",
"google/protobuf/util/type_resolver_util.h",
"google/protobuf/wrappers.pb.h",
"google/protobuf/wrappers.proto",
]
genrule(
name = "link_headers",
outs = HEADERS,
cmd = """
for i in $(OUTS); do
f=$${i#$(@D)/}
mkdir -p $(@D)/$${f%/*}
ln -sf $(INCLUDEDIR)/$$f $(@D)/$$f
done
""",
)
cc_library(
name = "protobuf",
hdrs = HEADERS,
linkopts = ["-lprotobuf"],
visibility = ["//visibility:public"],
)
cc_library(
name = "protobuf_headers",
hdrs = HEADERS,
linkopts = ["-lprotobuf"],
visibility = ["//visibility:public"],
)
cc_library(
name = "protoc_lib",
linkopts = ["-lprotoc"],
visibility = ["//visibility:public"],
)
genrule(
name = "protoc",
outs = ["protoc.bin"],
cmd = "ln -s $$(which protoc) $@",
executable = 1,
visibility = ["//visibility:public"],
)
cc_proto_library(
name = "cc_wkt_protos",
hdrs = HEADERS,
internal_bootstrap_hack = 1,
protoc = ":protoc",
visibility = ["//visibility:public"],
)
proto_gen(
name = "protobuf_python_genproto",
includes = ["."],
protoc = "@protobuf_archive//:protoc",
visibility = ["//visibility:public"],
)
py_library(
name = "protobuf_python",
data = [":link_headers"],
srcs_version = "PY2AND3",
visibility = ["//visibility:public"],
)

425
third_party/systemlibs/protobuf.bzl vendored Normal file
View File

@ -0,0 +1,425 @@
def _GetPath(ctx, path):
if ctx.label.workspace_root:
return ctx.label.workspace_root + "/" + path
else:
return path
def _IsNewExternal(ctx):
# Bazel 0.4.4 and older have genfiles paths that look like:
# bazel-out/local-fastbuild/genfiles/external/repo/foo
# After the exec root rearrangement, they look like:
# ../repo/bazel-out/local-fastbuild/genfiles/foo
return ctx.label.workspace_root.startswith("../")
def _GenDir(ctx):
if _IsNewExternal(ctx):
# We are using the fact that Bazel 0.4.4+ provides repository-relative paths
# for ctx.genfiles_dir.
return ctx.genfiles_dir.path + (
"/" + ctx.attr.includes[0] if ctx.attr.includes and ctx.attr.includes[0] else ""
)
# This means that we're either in the old version OR the new version in the local repo.
# Either way, appending the source path to the genfiles dir works.
return ctx.var["GENDIR"] + "/" + _SourceDir(ctx)
def _SourceDir(ctx):
if not ctx.attr.includes:
return ctx.label.workspace_root
if not ctx.attr.includes[0]:
return _GetPath(ctx, ctx.label.package)
if not ctx.label.package:
return _GetPath(ctx, ctx.attr.includes[0])
return _GetPath(ctx, ctx.label.package + "/" + ctx.attr.includes[0])
def _CcHdrs(srcs, use_grpc_plugin = False):
ret = [s[:-len(".proto")] + ".pb.h" for s in srcs]
if use_grpc_plugin:
ret += [s[:-len(".proto")] + ".grpc.pb.h" for s in srcs]
return ret
def _CcSrcs(srcs, use_grpc_plugin = False):
ret = [s[:-len(".proto")] + ".pb.cc" for s in srcs]
if use_grpc_plugin:
ret += [s[:-len(".proto")] + ".grpc.pb.cc" for s in srcs]
return ret
def _CcOuts(srcs, use_grpc_plugin = False):
return _CcHdrs(srcs, use_grpc_plugin) + _CcSrcs(srcs, use_grpc_plugin)
def _PyOuts(srcs, use_grpc_plugin = False):
ret = [s[:-len(".proto")] + "_pb2.py" for s in srcs]
if use_grpc_plugin:
ret += [s[:-len(".proto")] + "_pb2_grpc.py" for s in srcs]
return ret
def _RelativeOutputPath(path, include, dest = ""):
if include == None:
return path
if not path.startswith(include):
fail("Include path %s isn't part of the path %s." % (include, path))
if include and include[-1] != "/":
include = include + "/"
if dest and dest[-1] != "/":
dest = dest + "/"
path = path[len(include):]
return dest + path
def _proto_gen_impl(ctx):
"""General implementation for generating protos"""
srcs = ctx.files.srcs
deps = []
deps += ctx.files.srcs
source_dir = _SourceDir(ctx)
gen_dir = _GenDir(ctx)
if source_dir:
import_flags = ["-I" + source_dir, "-I" + gen_dir]
else:
import_flags = ["-I."]
for dep in ctx.attr.deps:
import_flags += dep.proto.import_flags
deps += dep.proto.deps
args = []
if ctx.attr.gen_cc:
args += ["--cpp_out=" + gen_dir]
if ctx.attr.gen_py:
args += ["--python_out=" + gen_dir]
inputs = srcs + deps
if ctx.executable.plugin:
plugin = ctx.executable.plugin
lang = ctx.attr.plugin_language
if not lang and plugin.basename.startswith("protoc-gen-"):
lang = plugin.basename[len("protoc-gen-"):]
if not lang:
fail("cannot infer the target language of plugin", "plugin_language")
outdir = gen_dir
if ctx.attr.plugin_options:
outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir
args += ["--plugin=protoc-gen-%s=%s" % (lang, plugin.path)]
args += ["--%s_out=%s" % (lang, outdir)]
inputs += [plugin]
if args:
ctx.action(
inputs = inputs,
outputs = ctx.outputs.outs,
arguments = args + import_flags + [s.path for s in srcs],
executable = ctx.executable.protoc,
mnemonic = "ProtoCompile",
use_default_shell_env = True,
)
return struct(
proto = struct(
srcs = srcs,
import_flags = import_flags,
deps = deps,
),
)
proto_gen = rule(
attrs = {
"srcs": attr.label_list(allow_files = True),
"deps": attr.label_list(providers = ["proto"]),
"includes": attr.string_list(),
"protoc": attr.label(
cfg = "host",
executable = True,
single_file = True,
mandatory = True,
),
"plugin": attr.label(
cfg = "host",
allow_files = True,
executable = True,
),
"plugin_language": attr.string(),
"plugin_options": attr.string_list(),
"gen_cc": attr.bool(),
"gen_py": attr.bool(),
"outs": attr.output_list(),
},
output_to_genfiles = True,
implementation = _proto_gen_impl,
)
"""Generates codes from Protocol Buffers definitions.
This rule helps you to implement Skylark macros specific to the target
language. You should prefer more specific `cc_proto_library `,
`py_proto_library` and others unless you are adding such wrapper macros.
Args:
srcs: Protocol Buffers definition files (.proto) to run the protocol compiler
against.
deps: a list of dependency labels; must be other proto libraries.
includes: a list of include paths to .proto files.
protoc: the label of the protocol compiler to generate the sources.
plugin: the label of the protocol compiler plugin to be passed to the protocol
compiler.
plugin_language: the language of the generated sources
plugin_options: a list of options to be passed to the plugin
gen_cc: generates C++ sources in addition to the ones from the plugin.
gen_py: generates Python sources in addition to the ones from the plugin.
outs: a list of labels of the expected outputs from the protocol compiler.
"""
def cc_proto_library(
name,
srcs = [],
deps = [],
cc_libs = [],
include = None,
protoc = "@com_google_protobuf//:protoc",
internal_bootstrap_hack = False,
use_grpc_plugin = False,
default_runtime = "@com_google_protobuf//:protobuf",
**kargs):
"""Bazel rule to create a C++ protobuf library from proto source files
NOTE: the rule is only an internal workaround to generate protos. The
interface may change and the rule may be removed when bazel has introduced
the native rule.
Args:
name: the name of the cc_proto_library.
srcs: the .proto files of the cc_proto_library.
deps: a list of dependency labels; must be cc_proto_library.
cc_libs: a list of other cc_library targets depended by the generated
cc_library.
include: a string indicating the include path of the .proto files.
protoc: the label of the protocol compiler to generate the sources.
internal_bootstrap_hack: a flag indicate the cc_proto_library is used only
for bootstraping. When it is set to True, no files will be generated.
The rule will simply be a provider for .proto files, so that other
cc_proto_library can depend on it.
use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin
when processing the proto files.
default_runtime: the implicitly default runtime which will be depended on by
the generated cc_library target.
**kargs: other keyword arguments that are passed to cc_library.
"""
includes = []
if include != None:
includes = [include]
if internal_bootstrap_hack:
# For pre-checked-in generated files, we add the internal_bootstrap_hack
# which will skip the codegen action.
proto_gen(
name = name + "_genproto",
srcs = srcs,
deps = [s + "_genproto" for s in deps],
includes = includes,
protoc = protoc,
visibility = ["//visibility:public"],
)
# An empty cc_library to make rule dependency consistent.
native.cc_library(
name = name,
**kargs
)
return
grpc_cpp_plugin = None
if use_grpc_plugin:
grpc_cpp_plugin = "//external:grpc_cpp_plugin"
gen_srcs = _CcSrcs(srcs, use_grpc_plugin)
gen_hdrs = _CcHdrs(srcs, use_grpc_plugin)
outs = gen_srcs + gen_hdrs
proto_gen(
name = name + "_genproto",
srcs = srcs,
deps = [s + "_genproto" for s in deps],
includes = includes,
protoc = protoc,
plugin = grpc_cpp_plugin,
plugin_language = "grpc",
gen_cc = 1,
outs = outs,
visibility = ["//visibility:public"],
)
if default_runtime and not default_runtime in cc_libs:
cc_libs = cc_libs + [default_runtime]
if use_grpc_plugin:
cc_libs = cc_libs + ["//external:grpc_lib"]
native.cc_library(
name = name,
srcs = gen_srcs,
hdrs = gen_hdrs,
deps = cc_libs + deps,
includes = includes,
**kargs
)
def internal_gen_well_known_protos_java(srcs):
"""Bazel rule to generate the gen_well_known_protos_java genrule
Args:
srcs: the well known protos
"""
root = Label("%s//protobuf_java" % (REPOSITORY_NAME)).workspace_root
pkg = PACKAGE_NAME + "/" if PACKAGE_NAME else ""
if root == "":
include = " -I%ssrc " % pkg
else:
include = " -I%s/%ssrc " % (root, pkg)
native.genrule(
name = "gen_well_known_protos_java",
srcs = srcs,
outs = [
"wellknown.srcjar",
],
cmd = "$(location :protoc) --java_out=$(@D)/wellknown.jar" +
" %s $(SRCS) " % include +
" && mv $(@D)/wellknown.jar $(@D)/wellknown.srcjar",
tools = [":protoc"],
)
def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs):
"""Macro to copy files to a different directory and then create a filegroup.
This is used by the //:protobuf_python py_proto_library target to work around
an issue caused by Python source files that are part of the same Python
package being in separate directories.
Args:
srcs: The source files to copy and add to the filegroup.
strip_prefix: Path to the root of the files to copy.
dest: The directory to copy the source files into.
**kwargs: extra arguments that will be passesd to the filegroup.
"""
outs = [_RelativeOutputPath(s, strip_prefix, dest) for s in srcs]
native.genrule(
name = name + "_genrule",
srcs = srcs,
outs = outs,
cmd = " && ".join(
["cp $(location %s) $(location %s)" %
(s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs],
),
)
native.filegroup(
name = name,
srcs = outs,
**kwargs
)
def py_proto_library(
name,
srcs = [],
deps = [],
py_libs = [],
py_extra_srcs = [],
include = None,
default_runtime = "@com_google_protobuf//:protobuf_python",
protoc = "@com_google_protobuf//:protoc",
use_grpc_plugin = False,
**kargs):
"""Bazel rule to create a Python protobuf library from proto source files
NOTE: the rule is only an internal workaround to generate protos. The
interface may change and the rule may be removed when bazel has introduced
the native rule.
Args:
name: the name of the py_proto_library.
srcs: the .proto files of the py_proto_library.
deps: a list of dependency labels; must be py_proto_library.
py_libs: a list of other py_library targets depended by the generated
py_library.
py_extra_srcs: extra source files that will be added to the output
py_library. This attribute is used for internal bootstrapping.
include: a string indicating the include path of the .proto files.
default_runtime: the implicitly default runtime which will be depended on by
the generated py_library target.
protoc: the label of the protocol compiler to generate the sources.
use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin
when processing the proto files.
**kargs: other keyword arguments that are passed to cc_library.
"""
outs = _PyOuts(srcs, use_grpc_plugin)
includes = []
if include != None:
includes = [include]
grpc_python_plugin = None
if use_grpc_plugin:
grpc_python_plugin = "//external:grpc_python_plugin"
# Note: Generated grpc code depends on Python grpc module. This dependency
# is not explicitly listed in py_libs. Instead, host system is assumed to
# have grpc installed.
proto_gen(
name = name + "_genproto",
srcs = srcs,
deps = [s + "_genproto" for s in deps],
includes = includes,
protoc = protoc,
gen_py = 1,
outs = outs,
visibility = ["//visibility:public"],
plugin = grpc_python_plugin,
plugin_language = "grpc",
)
if default_runtime and not default_runtime in py_libs + deps:
py_libs = py_libs + [default_runtime]
native.py_library(
name = name,
srcs = outs + py_extra_srcs,
deps = py_libs + deps,
imports = includes,
**kargs
)
def internal_protobuf_py_tests(
name,
modules = [],
**kargs):
"""Bazel rules to create batch tests for protobuf internal.
Args:
name: the name of the rule.
modules: a list of modules for tests. The macro will create a py_test for
each of the parameter with the source "google/protobuf/%s.py"
kargs: extra parameters that will be passed into the py_test.
"""
for m in modules:
s = "python/google/protobuf/internal/%s.py" % m
native.py_test(
name = "py_%s" % m,
srcs = [s],
main = s,
**kargs
)
def check_protobuf_required_bazel_version():
"""For WORKSPACE files, to check the installed version of bazel.
This ensures bazel supports our approach to proto_library() depending on a
copied filegroup. (Fixed in bazel 0.5.4)
"""
expected = apple_common.dotted_version("0.5.4")
current = apple_common.dotted_version(native.bazel_version)
if current.compare_to(expected) < 0:
fail("Bazel must be newer than 0.5.4")

View File

@ -15,6 +15,8 @@ VALID_LIBS = [
"boringssl",
"com_github_googleapis_googleapis",
"com_github_googlecloudplatform_google_cloud_cpp",
"com_google_protobuf",
"com_google_protobuf_cc",
"com_googlesource_code_re2",
"curl",
"cython",
@ -32,6 +34,7 @@ VALID_LIBS = [
"org_sqlite",
"pcre",
"png_archive",
"protobuf_archive",
"six_archive",
"snappy",
"swig",