experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update protobuf to v3.14.

Browse Source 
PiperOrigin-RevId: 353684435
Change-Id: Ia0bade180261ce9722ae2090162075d95e82cdc6
This commit is contained in:
Mihai Maruseac 2021-01-25 11:01:21 -08:00 committed by TensorFlower Gardener
parent 4e224e064d
commit 61fb67458e
4 changed files with 151 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tensorflow/core/platform/default/build_config.bzl
View File

@ -152,6 +152,7 @@ def cc_proto_library(
cc_libs = [], cc_libs = [],
include = None, include = None,
protoc = "@com_google_protobuf//:protoc", protoc = "@com_google_protobuf//:protoc",
internal_bootstrap_hack = False,
use_grpc_plugin = False, use_grpc_plugin = False,
use_grpc_namespace = False, use_grpc_namespace = False,
make_default_target_header_only = False, make_default_target_header_only = False,
@ -168,6 +169,10 @@ def cc_proto_library(
cc_library. cc_library.
include: a string indicating the include path of the .proto files. include: a string indicating the include path of the .proto files.
protoc: the label of the protocol compiler to generate the sources. protoc: the label of the protocol compiler to generate the sources.
internal_bootstrap_hack: a flag indicate the cc_proto_library is used only
for bootstraping. When it is set to True, no files will be generated.
The rule will simply be a provider for .proto files, so that other
cc_proto_library can depend on it.
use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin
when processing the proto files. when processing the proto files.
use_grpc_namespace: the namespace for the grpc services. use_grpc_namespace: the namespace for the grpc services.
@ -189,6 +194,25 @@ def cc_proto_library(
if protolib_name == None: if protolib_name == None:
protolib_name = name protolib_name = name
if internal_bootstrap_hack:
# For pre-checked-in generated files, we add the internal_bootstrap_hack
# which will skip the codegen action.
proto_gen(
name = protolib_name + "_genproto",
srcs = srcs,
includes = includes,
protoc = protoc,
visibility = ["//visibility:public"],
deps = [s + "_genproto" for s in all_protolib_deps],
)
# An empty cc_library to make rule dependency consistent.
native.cc_library(
name = name,
**kargs
)
return
grpc_cpp_plugin = None grpc_cpp_plugin = None
plugin_options = [] plugin_options = []
if use_grpc_plugin: if use_grpc_plugin:
@ -248,10 +272,10 @@ def cc_proto_library(
) )
native.cc_library( native.cc_library(
name = header_only_name, name = header_only_name,
hdrs = gen_hdrs,
deps = [ deps = [
"@com_google_protobuf//:protobuf_headers", "@com_google_protobuf//:protobuf_headers",
] + header_only_deps + if_static([impl_name]), ] + header_only_deps + if_static([impl_name]),
hdrs = gen_hdrs,
**kargs **kargs
) )

12
tensorflow/workspace.bzl
View File

@ -584,13 +584,15 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
tf_http_archive( tf_http_archive(
name = "com_google_protobuf", name = "com_google_protobuf",
patch_file = clean_dep("//third_party/protobuf:protobuf.patch"), patch_file = clean_dep("//third_party/protobuf:protobuf.patch"),
sha256 = "bf0e5070b4b99240183b29df78155eee335885e53a8af8683964579c214ad301", sha256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b",
strip_prefix = "protobuf-3.14.0", strip_prefix = "protobuf-3.9.2",
system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"), system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"),
system_link_files = {"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl"}, system_link_files = {
"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
},
urls = [ urls = [
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.14.0.zip", "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
"https://github.com/protocolbuffers/protobuf/archive/v3.14.0.zip", "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
], ],
) )

31
third_party/protobuf/protobuf.patch vendored
View File

@ -1,8 +1,8 @@
diff --git a/BUILD b/BUILD diff --git a/BUILD b/BUILD
index 112432160..4ba0da0ee 100644 index dbae719ff..87dc38470 100644
--- a/BUILD --- a/BUILD
+++ b/BUILD +++ b/BUILD
@@ -50,7 +50,7 @@ GTEST_MAIN = select({ @@ -23,7 +23,7 @@ config_setting(
# ZLIB configuration # ZLIB configuration
################################################################################ ################################################################################
@ -11,7 +11,7 @@ index 112432160..4ba0da0ee 100644
################################################################################ ################################################################################
# Protobuf Runtime Library # Protobuf Runtime Library
@@ -199,6 +199,7 @@ cc_library( @@ -143,6 +143,7 @@ cc_library(
copts = COPTS, copts = COPTS,
includes = ["src/"], includes = ["src/"],
linkopts = LINK_OPTS, linkopts = LINK_OPTS,
@ -19,7 +19,7 @@ index 112432160..4ba0da0ee 100644
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
@@ -271,6 +272,7 @@ cc_library( @@ -213,6 +214,7 @@ cc_library(
copts = COPTS, copts = COPTS,
includes = ["src/"], includes = ["src/"],
linkopts = LINK_OPTS, linkopts = LINK_OPTS,
@ -27,12 +27,17 @@ index 112432160..4ba0da0ee 100644
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = [":protobuf_lite"] + PROTOBUF_DEPS, deps = [":protobuf_lite"] + PROTOBUF_DEPS,
) )
@@ -867,7 +869,7 @@ py_proto_library( diff --git a/protobuf.bzl b/protobuf.bzl
protoc = ":protoc", index e0653321f..253d9cbb5 100644
py_libs = [ --- a/protobuf.bzl
":python_srcs", +++ b/protobuf.bzl
- "@six//:six", @@ -84,7 +84,9 @@ def _proto_gen_impl(ctx):
+ "//external:six",
], for dep in ctx.attr.deps:
srcs_version = "PY2AND3", import_flags += dep.proto.import_flags
visibility = ["//visibility:public"], deps += dep.proto.deps
+ import_flags = depset(import_flags).to_list()
+ deps = depset(deps).to_list()
if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin:
return struct(

224
third_party/systemlibs/protobuf.bzl vendored
View File

@ -1,8 +1,3 @@
load("@bazel_skylib//lib:versions.bzl", "versions")
load("@rules_cc//cc:defs.bzl", "cc_library")
load("@rules_proto//proto:defs.bzl", "ProtoInfo")
load("@rules_python//python:defs.bzl", "py_library", "py_test")
def _GetPath(ctx, path): def _GetPath(ctx, path):
if ctx.label.workspace_root: if ctx.label.workspace_root:
return ctx.label.workspace_root + "/" + path return ctx.label.workspace_root + "/" + path
@ -76,55 +71,28 @@ def _RelativeOutputPath(path, include, dest = ""):
def _proto_gen_impl(ctx): def _proto_gen_impl(ctx):
"""General implementation for generating protos""" """General implementation for generating protos"""
srcs = ctx.files.srcs srcs = ctx.files.srcs
deps = depset(direct = ctx.files.srcs) deps = []
deps += ctx.files.srcs
source_dir = _SourceDir(ctx) source_dir = _SourceDir(ctx)
gen_dir = _GenDir(ctx).rstrip("/") gen_dir = _GenDir(ctx)
if source_dir: if source_dir:
import_flags = depset(direct = ["-I" + source_dir, "-I" + gen_dir]) import_flags = ["-I" + source_dir, "-I" + gen_dir]
else: else:
import_flags = depset(direct = ["-I."]) import_flags = ["-I."]
for dep in ctx.attr.deps: for dep in ctx.attr.deps:
if type(dep.proto.import_flags) == "list": import_flags += dep.proto.import_flags
import_flags = depset(transitive = [import_flags], direct = dep.proto.import_flags) deps += dep.proto.deps
else: import_flags = depset(import_flags).to_list()
import_flags = depset(transitive = [import_flags, dep.proto.import_flags]) deps = depset(deps).to_list()
if type(dep.proto.deps) == "list":
deps = depset(transitive = [deps], direct = dep.proto.deps)
else:
deps = depset(transitive = [deps, dep.proto.deps])
if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin:
return struct(
proto = struct(
srcs = srcs,
import_flags = import_flags,
deps = deps,
),
)
for src in srcs:
args = [] args = []
in_gen_dir = src.root.path == gen_dir
if in_gen_dir:
import_flags_real = []
for f in import_flags.to_list():
path = f.replace("-I", "")
import_flags_real.append("-I$(realpath -s %s)" % path)
outs = []
use_grpc_plugin = (ctx.attr.plugin_language == "grpc" and ctx.attr.plugin)
path_tpl = "$(realpath %s)" if in_gen_dir else "%s"
if ctx.attr.gen_cc: if ctx.attr.gen_cc:
args += [("--cpp_out=" + path_tpl) % gen_dir] args += ["--cpp_out=" + gen_dir]
outs.extend(_CcOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
if ctx.attr.gen_py: if ctx.attr.gen_py:
args += [("--python_out=" + path_tpl) % gen_dir] args += ["--python_out=" + gen_dir]
outs.extend(_PyOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
outs = [ctx.actions.declare_file(out, sibling = src) for out in outs] inputs = srcs + deps
inputs = [src] + deps.to_list()
tools = [ctx.executable.protoc] tools = [ctx.executable.protoc]
if ctx.executable.plugin: if ctx.executable.plugin:
plugin = ctx.executable.plugin plugin = ctx.executable.plugin
@ -134,44 +102,20 @@ def _proto_gen_impl(ctx):
if not lang: if not lang:
fail("cannot infer the target language of plugin", "plugin_language") fail("cannot infer the target language of plugin", "plugin_language")
outdir = "." if in_gen_dir else gen_dir outdir = gen_dir
if ctx.attr.plugin_options: if ctx.attr.plugin_options:
outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir
args += [("--plugin=protoc-gen-%s=" + path_tpl) % (lang, plugin.path)] args += ["--plugin=protoc-gen-%s=%s" % (lang, plugin.path)]
args += ["--%s_out=%s" % (lang, outdir)] args += ["--%s_out=%s" % (lang, outdir)]
tools.append(plugin) tools.append(plugin)
if not in_gen_dir: if args:
ctx.actions.run( ctx.actions.run(
inputs = inputs, inputs = inputs,
tools = tools, outputs = ctx.outputs.outs,
outputs = outs, arguments = args + import_flags + [s.path for s in srcs],
arguments = args + import_flags.to_list() + [src.path],
executable = ctx.executable.protoc, executable = ctx.executable.protoc,
mnemonic = "ProtoCompile", mnemonic = "ProtoCompile",
use_default_shell_env = True,
)
else:
for out in outs:
orig_command = " ".join(
["$(realpath %s)" % ctx.executable.protoc.path] + args +
import_flags_real + ["-I.", src.basename],
)
command = ";".join([
'CMD="%s"' % orig_command,
"cd %s" % src.dirname,
"${CMD}",
"cd -",
])
generated_out = "/".join([gen_dir, out.basename])
if generated_out != out.path:
command += ";mv %s %s" % (generated_out, out.path)
ctx.actions.run_shell(
inputs = inputs,
outputs = [out],
command = command,
mnemonic = "ProtoCompile",
tools = tools, tools = tools,
use_default_shell_env = True, use_default_shell_env = True,
) )
@ -230,29 +174,6 @@ Args:
outs: a list of labels of the expected outputs from the protocol compiler. outs: a list of labels of the expected outputs from the protocol compiler.
""" """
def _adapt_proto_library_impl(ctx):
deps = [dep[ProtoInfo] for dep in ctx.attr.deps]
srcs = [src for dep in deps for src in dep.direct_sources]
return struct(
proto = struct(
srcs = srcs,
import_flags = ["-I{}".format(path) for dep in deps for path in dep.transitive_proto_path.to_list()],
deps = srcs,
),
)
adapt_proto_library = rule(
implementation = _adapt_proto_library_impl,
attrs = {
"deps": attr.label_list(
mandatory = True,
providers = [ProtoInfo],
),
},
doc = "Adapts `proto_library` from `@rules_proto` to be used with `{cc,py}_proto_library` from this file.",
)
def cc_proto_library( def cc_proto_library(
name, name,
srcs = [], srcs = [],
@ -260,6 +181,7 @@ def cc_proto_library(
cc_libs = [], cc_libs = [],
include = None, include = None,
protoc = "@com_google_protobuf//:protoc", protoc = "@com_google_protobuf//:protoc",
internal_bootstrap_hack = False,
use_grpc_plugin = False, use_grpc_plugin = False,
default_runtime = "@com_google_protobuf//:protobuf", default_runtime = "@com_google_protobuf//:protobuf",
**kargs): **kargs):
@ -277,17 +199,41 @@ def cc_proto_library(
cc_library. cc_library.
include: a string indicating the include path of the .proto files. include: a string indicating the include path of the .proto files.
protoc: the label of the protocol compiler to generate the sources. protoc: the label of the protocol compiler to generate the sources.
internal_bootstrap_hack: a flag indicate the cc_proto_library is used only
for bootstraping. When it is set to True, no files will be generated.
The rule will simply be a provider for .proto files, so that other
cc_proto_library can depend on it.
use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin
when processing the proto files. when processing the proto files.
default_runtime: the implicitly default runtime which will be depended on by default_runtime: the implicitly default runtime which will be depended on by
the generated cc_library target. the generated cc_library target.
**kargs: other keyword arguments that are passed to cc_library. **kargs: other keyword arguments that are passed to cc_library.
""" """
includes = [] includes = []
if include != None: if include != None:
includes = [include] includes = [include]
if internal_bootstrap_hack:
# For pre-checked-in generated files, we add the internal_bootstrap_hack
# which will skip the codegen action.
proto_gen(
name = name + "_genproto",
srcs = srcs,
deps = [s + "_genproto" for s in deps],
includes = includes,
protoc = protoc,
visibility = ["//visibility:public"],
)
# An empty cc_library to make rule dependency consistent.
native.cc_library(
name = name,
**kargs
)
return
grpc_cpp_plugin = None grpc_cpp_plugin = None
if use_grpc_plugin: if use_grpc_plugin:
grpc_cpp_plugin = "//external:grpc_cpp_plugin" grpc_cpp_plugin = "//external:grpc_cpp_plugin"
@ -313,67 +259,39 @@ def cc_proto_library(
cc_libs = cc_libs + [default_runtime] cc_libs = cc_libs + [default_runtime]
if use_grpc_plugin: if use_grpc_plugin:
cc_libs = cc_libs + ["//external:grpc_lib"] cc_libs = cc_libs + ["//external:grpc_lib"]
cc_library(
native.cc_library(
name = name, name = name,
srcs = gen_srcs, srcs = gen_srcs,
hdrs = gen_hdrs, hdrs = gen_hdrs,
deps = cc_libs + deps, deps = cc_libs + deps,
includes = includes, includes = includes,
alwayslink = 1,
**kargs **kargs
) )
def _internal_gen_well_known_protos_java_impl(ctx): def internal_gen_well_known_protos_java(srcs):
args = ctx.actions.args() """Bazel rule to generate the gen_well_known_protos_java genrule
deps = [d[ProtoInfo] for d in ctx.attr.deps] Args:
srcs: the well known protos
srcjar = ctx.actions.declare_file("{}.srcjar".format(ctx.attr.name)) """
args.add("--java_out", srcjar) root = Label("%s//protobuf_java" % (native.repository_name())).workspace_root
pkg = native.package_name() + "/" if native.package_name() else ""
descriptors = depset( if root == "":
transitive = [dep.transitive_descriptor_sets for dep in deps], include = " -I%ssrc " % pkg
)
args.add_joined(
"--descriptor_set_in",
descriptors,
join_with = ctx.configuration.host_path_separator,
)
for dep in deps:
if "." == dep.proto_source_root:
args.add_all([src.path for src in dep.direct_sources])
else: else:
source_root = dep.proto_source_root include = " -I%s/%ssrc " % (root, pkg)
offset = len(source_root) + 1 # + '/'. native.genrule(
args.add_all([src.path[offset:] for src in dep.direct_sources]) name = "gen_well_known_protos_java",
srcs = srcs,
ctx.actions.run( outs = [
executable = ctx.executable._protoc, "wellknown.srcjar",
inputs = descriptors, ],
outputs = [srcjar], cmd = "$(location :protoc) --java_out=$(@D)/wellknown.jar" +
arguments = [args], " %s $(SRCS) " % include +
use_default_shell_env = True, " && mv $(@D)/wellknown.jar $(@D)/wellknown.srcjar",
) tools = [":protoc"],
return [
DefaultInfo(
files = depset([srcjar]),
),
]
internal_gen_well_known_protos_java = rule(
implementation = _internal_gen_well_known_protos_java_impl,
attrs = {
"deps": attr.label_list(
mandatory = True,
providers = [ProtoInfo],
),
"_protoc": attr.label(
executable = True,
cfg = "host",
default = "@com_google_protobuf//:protoc",
),
},
) )
def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs): def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs):
@ -438,7 +356,7 @@ def py_proto_library(
protoc: the label of the protocol compiler to generate the sources. protoc: the label of the protocol compiler to generate the sources.
use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin
when processing the proto files. when processing the proto files.
**kargs: other keyword arguments that are passed to py_library. **kargs: other keyword arguments that are passed to cc_library.
""" """
outs = _PyOuts(srcs, use_grpc_plugin) outs = _PyOuts(srcs, use_grpc_plugin)
@ -469,7 +387,8 @@ def py_proto_library(
if default_runtime and not default_runtime in py_libs + deps: if default_runtime and not default_runtime in py_libs + deps:
py_libs = py_libs + [default_runtime] py_libs = py_libs + [default_runtime]
py_library(
native.py_library(
name = name, name = name,
srcs = outs + py_extra_srcs, srcs = outs + py_extra_srcs,
deps = py_libs + deps, deps = py_libs + deps,
@ -492,7 +411,7 @@ def internal_protobuf_py_tests(
""" """
for m in modules: for m in modules:
s = "python/google/protobuf/internal/%s.py" % m s = "python/google/protobuf/internal/%s.py" % m
py_test( native.py_test(
name = "py_%s" % m, name = "py_%s" % m,
srcs = [s], srcs = [s],
main = s, main = s,
@ -505,4 +424,7 @@ def check_protobuf_required_bazel_version():
This ensures bazel supports our approach to proto_library() depending on a This ensures bazel supports our approach to proto_library() depending on a
copied filegroup. (Fixed in bazel 0.5.4) copied filegroup. (Fixed in bazel 0.5.4)
""" """
versions.check(minimum_bazel_version = "0.5.4") expected = apple_common.dotted_version("0.5.4")
current = apple_common.dotted_version(native.bazel_version)
if current.compare_to(expected) < 0:
fail("Bazel must be newer than 0.5.4")