experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update protobuf to v3.14.

Browse Source 
PiperOrigin-RevId: 353684435
Change-Id: Ia0bade180261ce9722ae2090162075d95e82cdc6
This commit is contained in:
Mihai Maruseac 2021-01-25 11:01:21 -08:00 committed by TensorFlower Gardener
parent 4e224e064d
commit 61fb67458e
4 changed files with 151 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
tensorflow/core/platform/default/build_config.bzl
View File

@ -152,6 +152,7 @@ def cc_proto_library(
cc_libs = [],
include = None,
protoc = "@com_google_protobuf//:protoc",
internal_bootstrap_hack = False,
use_grpc_plugin = False,
use_grpc_namespace = False,
make_default_target_header_only = False,
@ -168,6 +169,10 @@ def cc_proto_library(
cc_library.
include: a string indicating the include path of the .proto files.
protoc: the label of the protocol compiler to generate the sources.
internal_bootstrap_hack: a flag indicate the cc_proto_library is used only
for bootstraping. When it is set to True, no files will be generated.
The rule will simply be a provider for .proto files, so that other
cc_proto_library can depend on it.
use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin
when processing the proto files.
use_grpc_namespace: the namespace for the grpc services.
@ -189,6 +194,25 @@ def cc_proto_library(
if protolib_name == None:
protolib_name = name
if internal_bootstrap_hack:
# For pre-checked-in generated files, we add the internal_bootstrap_hack
# which will skip the codegen action.
proto_gen(
name = protolib_name + "_genproto",
srcs = srcs,
includes = includes,
protoc = protoc,
visibility = ["//visibility:public"],
deps = [s + "_genproto" for s in all_protolib_deps],
)
# An empty cc_library to make rule dependency consistent.
native.cc_library(
name = name,
**kargs
)
return
grpc_cpp_plugin = None
plugin_options = []
if use_grpc_plugin:
@ -248,10 +272,10 @@ def cc_proto_library(
)
native.cc_library(
name = header_only_name,
hdrs = gen_hdrs,
deps = [
"@com_google_protobuf//:protobuf_headers",
] + header_only_deps + if_static([impl_name]),
hdrs = gen_hdrs,
**kargs
)

12
tensorflow/workspace.bzl
View File

@ -584,13 +584,15 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""):
tf_http_archive(
name = "com_google_protobuf",
patch_file = clean_dep("//third_party/protobuf:protobuf.patch"),
sha256 = "bf0e5070b4b99240183b29df78155eee335885e53a8af8683964579c214ad301",
strip_prefix = "protobuf-3.14.0",
sha256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b",
strip_prefix = "protobuf-3.9.2",
system_build_file = clean_dep("//third_party/systemlibs:protobuf.BUILD"),
system_link_files = {"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl"},
system_link_files = {
"//third_party/systemlibs:protobuf.bzl": "protobuf.bzl",
},
urls = [
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.14.0.zip",
"https://github.com/protocolbuffers/protobuf/archive/v3.14.0.zip",
"https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
"https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
],
)

31
third_party/protobuf/protobuf.patch vendored
View File

@ -1,8 +1,8 @@
diff --git a/BUILD b/BUILD
index 112432160..4ba0da0ee 100644
index dbae719ff..87dc38470 100644
--- a/BUILD
+++ b/BUILD
@@ -50,7 +50,7 @@ GTEST_MAIN = select({
@@ -23,7 +23,7 @@ config_setting(
# ZLIB configuration
################################################################################
@ -11,7 +11,7 @@ index 112432160..4ba0da0ee 100644
################################################################################
# Protobuf Runtime Library
@@ -199,6 +199,7 @@ cc_library(
@@ -143,6 +143,7 @@ cc_library(
copts = COPTS,
includes = ["src/"],
linkopts = LINK_OPTS,
@ -19,7 +19,7 @@ index 112432160..4ba0da0ee 100644
visibility = ["//visibility:public"],
)
@@ -271,6 +272,7 @@ cc_library(
@@ -213,6 +214,7 @@ cc_library(
copts = COPTS,
includes = ["src/"],
linkopts = LINK_OPTS,
@ -27,12 +27,17 @@ index 112432160..4ba0da0ee 100644
visibility = ["//visibility:public"],
deps = [":protobuf_lite"] + PROTOBUF_DEPS,
)
@@ -867,7 +869,7 @@ py_proto_library(
protoc = ":protoc",
py_libs = [
":python_srcs",
- "@six//:six",
+ "//external:six",
],
srcs_version = "PY2AND3",
visibility = ["//visibility:public"],
diff --git a/protobuf.bzl b/protobuf.bzl
index e0653321f..253d9cbb5 100644
--- a/protobuf.bzl
+++ b/protobuf.bzl
@@ -84,7 +84,9 @@ def _proto_gen_impl(ctx):
for dep in ctx.attr.deps:
import_flags += dep.proto.import_flags
deps += dep.proto.deps
+ import_flags = depset(import_flags).to_list()
+ deps = depset(deps).to_list()
if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin:
return struct(

274
third_party/systemlibs/protobuf.bzl vendored
View File

@ -1,8 +1,3 @@
load("@bazel_skylib//lib:versions.bzl", "versions")
load("@rules_cc//cc:defs.bzl", "cc_library")
load("@rules_proto//proto:defs.bzl", "ProtoInfo")
load("@rules_python//python:defs.bzl", "py_library", "py_test")
def _GetPath(ctx, path):
if ctx.label.workspace_root:
return ctx.label.workspace_root + "/" + path
@ -76,106 +71,55 @@ def _RelativeOutputPath(path, include, dest = ""):
def _proto_gen_impl(ctx):
"""General implementation for generating protos"""
srcs = ctx.files.srcs
deps = depset(direct = ctx.files.srcs)
deps = []
deps += ctx.files.srcs
source_dir = _SourceDir(ctx)
gen_dir = _GenDir(ctx).rstrip("/")
gen_dir = _GenDir(ctx)
if source_dir:
import_flags = depset(direct = ["-I" + source_dir, "-I" + gen_dir])
import_flags = ["-I" + source_dir, "-I" + gen_dir]
else:
import_flags = depset(direct = ["-I."])
import_flags = ["-I."]
for dep in ctx.attr.deps:
if type(dep.proto.import_flags) == "list":
import_flags = depset(transitive = [import_flags], direct = dep.proto.import_flags)
else:
import_flags = depset(transitive = [import_flags, dep.proto.import_flags])
if type(dep.proto.deps) == "list":
deps = depset(transitive = [deps], direct = dep.proto.deps)
else:
deps = depset(transitive = [deps, dep.proto.deps])
import_flags += dep.proto.import_flags
deps += dep.proto.deps
import_flags = depset(import_flags).to_list()
deps = depset(deps).to_list()
if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin:
return struct(
proto = struct(
srcs = srcs,
import_flags = import_flags,
deps = deps,
),
args = []
if ctx.attr.gen_cc:
args += ["--cpp_out=" + gen_dir]
if ctx.attr.gen_py:
args += ["--python_out=" + gen_dir]
inputs = srcs + deps
tools = [ctx.executable.protoc]
if ctx.executable.plugin:
plugin = ctx.executable.plugin
lang = ctx.attr.plugin_language
if not lang and plugin.basename.startswith("protoc-gen-"):
lang = plugin.basename[len("protoc-gen-"):]
if not lang:
fail("cannot infer the target language of plugin", "plugin_language")
outdir = gen_dir
if ctx.attr.plugin_options:
outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir
args += ["--plugin=protoc-gen-%s=%s" % (lang, plugin.path)]
args += ["--%s_out=%s" % (lang, outdir)]
tools.append(plugin)
if args:
ctx.actions.run(
inputs = inputs,
outputs = ctx.outputs.outs,
arguments = args + import_flags + [s.path for s in srcs],
executable = ctx.executable.protoc,
mnemonic = "ProtoCompile",
tools = tools,
use_default_shell_env = True,
)
for src in srcs:
args = []
in_gen_dir = src.root.path == gen_dir
if in_gen_dir:
import_flags_real = []
for f in import_flags.to_list():
path = f.replace("-I", "")
import_flags_real.append("-I$(realpath -s %s)" % path)
outs = []
use_grpc_plugin = (ctx.attr.plugin_language == "grpc" and ctx.attr.plugin)
path_tpl = "$(realpath %s)" if in_gen_dir else "%s"
if ctx.attr.gen_cc:
args += [("--cpp_out=" + path_tpl) % gen_dir]
outs.extend(_CcOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
if ctx.attr.gen_py:
args += [("--python_out=" + path_tpl) % gen_dir]
outs.extend(_PyOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
outs = [ctx.actions.declare_file(out, sibling = src) for out in outs]
inputs = [src] + deps.to_list()
tools = [ctx.executable.protoc]
if ctx.executable.plugin:
plugin = ctx.executable.plugin
lang = ctx.attr.plugin_language
if not lang and plugin.basename.startswith("protoc-gen-"):
lang = plugin.basename[len("protoc-gen-"):]
if not lang:
fail("cannot infer the target language of plugin", "plugin_language")
outdir = "." if in_gen_dir else gen_dir
if ctx.attr.plugin_options:
outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir
args += [("--plugin=protoc-gen-%s=" + path_tpl) % (lang, plugin.path)]
args += ["--%s_out=%s" % (lang, outdir)]
tools.append(plugin)
if not in_gen_dir:
ctx.actions.run(
inputs = inputs,
tools = tools,
outputs = outs,
arguments = args + import_flags.to_list() + [src.path],
executable = ctx.executable.protoc,
mnemonic = "ProtoCompile",
use_default_shell_env = True,
)
else:
for out in outs:
orig_command = " ".join(
["$(realpath %s)" % ctx.executable.protoc.path] + args +
import_flags_real + ["-I.", src.basename],
)
command = ";".join([
'CMD="%s"' % orig_command,
"cd %s" % src.dirname,
"${CMD}",
"cd -",
])
generated_out = "/".join([gen_dir, out.basename])
if generated_out != out.path:
command += ";mv %s %s" % (generated_out, out.path)
ctx.actions.run_shell(
inputs = inputs,
outputs = [out],
command = command,
mnemonic = "ProtoCompile",
tools = tools,
use_default_shell_env = True,
)
return struct(
proto = struct(
srcs = srcs,
@ -230,29 +174,6 @@ Args:
outs: a list of labels of the expected outputs from the protocol compiler.
"""
def _adapt_proto_library_impl(ctx):
deps = [dep[ProtoInfo] for dep in ctx.attr.deps]
srcs = [src for dep in deps for src in dep.direct_sources]
return struct(
proto = struct(
srcs = srcs,
import_flags = ["-I{}".format(path) for dep in deps for path in dep.transitive_proto_path.to_list()],
deps = srcs,
),
)
adapt_proto_library = rule(
implementation = _adapt_proto_library_impl,
attrs = {
"deps": attr.label_list(
mandatory = True,
providers = [ProtoInfo],
),
},
doc = "Adapts `proto_library` from `@rules_proto` to be used with `{cc,py}_proto_library` from this file.",
)
def cc_proto_library(
name,
srcs = [],
@ -260,6 +181,7 @@ def cc_proto_library(
cc_libs = [],
include = None,
protoc = "@com_google_protobuf//:protoc",
internal_bootstrap_hack = False,
use_grpc_plugin = False,
default_runtime = "@com_google_protobuf//:protobuf",
**kargs):
@ -277,17 +199,41 @@ def cc_proto_library(
cc_library.
include: a string indicating the include path of the .proto files.
protoc: the label of the protocol compiler to generate the sources.
internal_bootstrap_hack: a flag indicate the cc_proto_library is used only
for bootstraping. When it is set to True, no files will be generated.
The rule will simply be a provider for .proto files, so that other
cc_proto_library can depend on it.
use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin
when processing the proto files.
default_runtime: the implicitly default runtime which will be depended on by
the generated cc_library target.
**kargs: other keyword arguments that are passed to cc_library.
"""
includes = []
if include != None:
includes = [include]
if internal_bootstrap_hack:
# For pre-checked-in generated files, we add the internal_bootstrap_hack
# which will skip the codegen action.
proto_gen(
name = name + "_genproto",
srcs = srcs,
deps = [s + "_genproto" for s in deps],
includes = includes,
protoc = protoc,
visibility = ["//visibility:public"],
)
# An empty cc_library to make rule dependency consistent.
native.cc_library(
name = name,
**kargs
)
return
grpc_cpp_plugin = None
if use_grpc_plugin:
grpc_cpp_plugin = "//external:grpc_cpp_plugin"
@ -313,68 +259,40 @@ def cc_proto_library(
cc_libs = cc_libs + [default_runtime]
if use_grpc_plugin:
cc_libs = cc_libs + ["//external:grpc_lib"]
cc_library(
native.cc_library(
name = name,
srcs = gen_srcs,
hdrs = gen_hdrs,
deps = cc_libs + deps,
includes = includes,
alwayslink = 1,
**kargs
)
def _internal_gen_well_known_protos_java_impl(ctx):
args = ctx.actions.args()
def internal_gen_well_known_protos_java(srcs):
"""Bazel rule to generate the gen_well_known_protos_java genrule
deps = [d[ProtoInfo] for d in ctx.attr.deps]
srcjar = ctx.actions.declare_file("{}.srcjar".format(ctx.attr.name))
args.add("--java_out", srcjar)
descriptors = depset(
transitive = [dep.transitive_descriptor_sets for dep in deps],
Args:
srcs: the well known protos
"""
root = Label("%s//protobuf_java" % (native.repository_name())).workspace_root
pkg = native.package_name() + "/" if native.package_name() else ""
if root == "":
include = " -I%ssrc " % pkg
else:
include = " -I%s/%ssrc " % (root, pkg)
native.genrule(
name = "gen_well_known_protos_java",
srcs = srcs,
outs = [
"wellknown.srcjar",
],
cmd = "$(location :protoc) --java_out=$(@D)/wellknown.jar" +
" %s $(SRCS) " % include +
" && mv $(@D)/wellknown.jar $(@D)/wellknown.srcjar",
tools = [":protoc"],
)
args.add_joined(
"--descriptor_set_in",
descriptors,
join_with = ctx.configuration.host_path_separator,
)
for dep in deps:
if "." == dep.proto_source_root:
args.add_all([src.path for src in dep.direct_sources])
else:
source_root = dep.proto_source_root
offset = len(source_root) + 1 # + '/'.
args.add_all([src.path[offset:] for src in dep.direct_sources])
ctx.actions.run(
executable = ctx.executable._protoc,
inputs = descriptors,
outputs = [srcjar],
arguments = [args],
use_default_shell_env = True,
)
return [
DefaultInfo(
files = depset([srcjar]),
),
]
internal_gen_well_known_protos_java = rule(
implementation = _internal_gen_well_known_protos_java_impl,
attrs = {
"deps": attr.label_list(
mandatory = True,
providers = [ProtoInfo],
),
"_protoc": attr.label(
executable = True,
cfg = "host",
default = "@com_google_protobuf//:protoc",
),
},
)
def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs):
"""Macro to copy files to a different directory and then create a filegroup.
@ -438,7 +356,7 @@ def py_proto_library(
protoc: the label of the protocol compiler to generate the sources.
use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin
when processing the proto files.
**kargs: other keyword arguments that are passed to py_library.
**kargs: other keyword arguments that are passed to cc_library.
"""
outs = _PyOuts(srcs, use_grpc_plugin)
@ -469,7 +387,8 @@ def py_proto_library(
if default_runtime and not default_runtime in py_libs + deps:
py_libs = py_libs + [default_runtime]
py_library(
native.py_library(
name = name,
srcs = outs + py_extra_srcs,
deps = py_libs + deps,
@ -492,7 +411,7 @@ def internal_protobuf_py_tests(
"""
for m in modules:
s = "python/google/protobuf/internal/%s.py" % m
py_test(
native.py_test(
name = "py_%s" % m,
srcs = [s],
main = s,
@ -505,4 +424,7 @@ def check_protobuf_required_bazel_version():
This ensures bazel supports our approach to proto_library() depending on a
copied filegroup. (Fixed in bazel 0.5.4)
"""
versions.check(minimum_bazel_version = "0.5.4")
expected = apple_common.dotted_version("0.5.4")
current = apple_common.dotted_version(native.bazel_version)
if current.compare_to(expected) < 0:
fail("Bazel must be newer than 0.5.4")