experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Convert TensorFlow's flatbuffers dependency to new third party import method

Browse Source 
PiperOrigin-RevId: 209833197
This commit is contained in:
Austin Anderson 2018-08-22 14:43:45 -07:00 committed by TensorFlower Gardener
parent ce127f779d
commit 13a4688fc3
18 changed files with 230 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tensorflow/contrib/lite/delegates/eager/kernel.cc
View File

@ -14,7 +14,7 @@ limitations under the License.
==============================================================================*/
#include "tensorflow/contrib/lite/delegates/eager/kernel.h"
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/builtin_ops.h"
#include "tensorflow/contrib/lite/context.h"
#include "tensorflow/contrib/lite/context_util.h"

2
tensorflow/contrib/lite/delegates/eager/test_util.cc
View File

@ -16,7 +16,7 @@ limitations under the License.
#include "tensorflow/contrib/lite/delegates/eager/test_util.h"
#include "absl/memory/memory.h"
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/string.h"
namespace tflite {

2
tensorflow/contrib/lite/experimental/kernels/ctc_beam_search_decoder.cc
View File

@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <vector>
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/context.h"
#include "tensorflow/contrib/lite/experimental/kernels/ctc_beam_search.h"
#include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h"

2
tensorflow/contrib/lite/experimental/kernels/ctc_beam_search_decoder_test.cc
View File

@ -18,7 +18,7 @@ limitations under the License.
#include <vector>
#include <gtest/gtest.h>
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/interpreter.h"
#include "tensorflow/contrib/lite/kernels/register.h"
#include "tensorflow/contrib/lite/kernels/test_util.h"

2
tensorflow/contrib/lite/kernels/audio_spectrogram.cc
View File

@ -22,7 +22,7 @@ limitations under the License.
#include "tensorflow/contrib/lite/kernels/kernel_util.h"
#include "tensorflow/contrib/lite/kernels/op_macros.h"
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
namespace tflite {
namespace ops {

2
tensorflow/contrib/lite/kernels/audio_spectrogram_test.cc
View File

@ -18,7 +18,7 @@ limitations under the License.
#include <vector>
#include <gtest/gtest.h>
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/interpreter.h"
#include "tensorflow/contrib/lite/kernels/register.h"
#include "tensorflow/contrib/lite/kernels/test_util.h"

2
tensorflow/contrib/lite/kernels/detection_postprocess.cc
View File

@ -15,7 +15,7 @@ limitations under the License.
#include <string.h>
#include <numeric>
#include <vector>
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/builtin_op_data.h"
#include "tensorflow/contrib/lite/context.h"
#include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h"

2
tensorflow/contrib/lite/kernels/detection_postprocess_test.cc
View File

@ -17,7 +17,7 @@ limitations under the License.
#include <vector>
#include <gtest/gtest.h>
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/interpreter.h"
#include "tensorflow/contrib/lite/kernels/register.h"
#include "tensorflow/contrib/lite/kernels/test_util.h"

2
tensorflow/contrib/lite/kernels/mfcc.cc
View File

@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/contrib/lite/kernels/internal/mfcc.h"
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/builtin_op_data.h"
#include "tensorflow/contrib/lite/context.h"
#include "tensorflow/contrib/lite/kernels/internal/mfcc_dct.h"

2
tensorflow/contrib/lite/kernels/mfcc_test.cc
View File

@ -18,7 +18,7 @@ limitations under the License.
#include <vector>
#include <gtest/gtest.h>
#include "flatbuffers/flexbuffers.h"
#include "flatbuffers/flexbuffers.h" // flatbuffers
#include "tensorflow/contrib/lite/interpreter.h"
#include "tensorflow/contrib/lite/kernels/register.h"
#include "tensorflow/contrib/lite/kernels/test_util.h"

2
tensorflow/contrib/lite/schema/BUILD
View File

@ -48,7 +48,7 @@ exports_files([
"schema_v3.fbs",
])
load("//third_party/flatbuffers:build_defs.bzl", "flatbuffer_cc_library")
load("@flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
# Generic schema for inference on device.
flatbuffer_cc_library(

2
tensorflow/contrib/lite/schema/flatbuffer_compatibility_test.cc
View File

@ -15,7 +15,7 @@ limitations under the License.
#include <fstream>
#include <gtest/gtest.h>
#include "flatbuffers/flatc.h"
#include "flatbuffers/flatc.h" // flatbuffers
#include "tensorflow/core/platform/platform.h"
#ifdef PLATFORM_GOOGLE

16
tensorflow/workspace.bzl
View File

@ -19,10 +19,10 @@ load(
"//tensorflow/tools/def_file_filter:def_file_filter_configure.bzl",
"def_file_filter_configure",
)
load("//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo")
def initialize_third_party():
# Fill in later
pass
flatbuffers()
# Sanitize a dependency so that it works correctly from code that includes
# TensorFlow as a submodule.
@ -740,18 +740,6 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"),
)
tf_http_archive(
name = "flatbuffers",
strip_prefix = "flatbuffers-1.9.0",
sha256 = "5ca5491e4260cacae30f1a5786d109230db3f3a6e5a0eb45d0d0608293d247e3",
urls = [
"https://mirror.bazel.build/github.com/google/flatbuffers/archive/v1.9.0.tar.gz",
"https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz",
],
build_file = clean_dep("//third_party/flatbuffers:flatbuffers.BUILD"),
system_build_file = clean_dep("//third_party/systemlibs:flatbuffers.BUILD"),
)
native.new_http_archive(
name = "double_conversion",
urls = [

16
third_party/flatbuffers/BUILD vendored
View File

@ -1,15 +1 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"]) # Apache 2.0
filegroup(
name = "all_files",
srcs = glob(
["**/*"],
exclude = [
"**/METADATA",
"**/OWNERS",
],
),
visibility = ["//tensorflow:__subpackages__"],
)
# This empty BUILD file is required to make Bazel treat this directory as a package.

5
third_party/flatbuffers/flatbuffers.BUILD → third_party/flatbuffers/BUILD.bazel vendored
View File

@ -19,7 +19,10 @@ config_setting(
FLATBUFFERS_COPTS = select({
":windows": [],
"//conditions:default": ["-Wno-implicit-fallthrough", "-fexceptions"],
"//conditions:default": [
"-Wno-implicit-fallthrough",
"-fexceptions",
],
})
# Public flatc library to compile flatbuffer files at runtime.

0
third_party/systemlibs/flatbuffers.BUILD → third_party/flatbuffers/BUILD.system vendored
View File

366
third_party/flatbuffers/build_defs.bzl vendored
View File

@ -1,5 +1,4 @@
# Description:
# BUILD rules for generating flatbuffer files.
"""BUILD rules for generating flatbuffer files."""
flatc_path = "@flatbuffers//:flatc"
@ -8,66 +7,50 @@ DEFAULT_FLATC_ARGS = [
"--gen-object-api",
]
def flatbuffer_library_public(name,
srcs,
outs,
language_flag,
out_prefix="",
includes=[],
include_paths=[],
flatc_args=DEFAULT_FLATC_ARGS,
reflection_name="",
reflection_visiblity=None,
output_to_bindir=False):
'''Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler.
def flatbuffer_library_public(
name,
srcs,
outs,
language_flag,
out_prefix = "",
includes = [],
include_paths = [],
flatc_args = DEFAULT_FLATC_ARGS,
reflection_name = "",
reflection_visiblity = None,
output_to_bindir = False):
"""Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler.
Args:
name: Rule name.
srcs: Source .fbs files. Sent in order to the compiler.
outs: Output files from flatc.
language_flag: Target language flag. One of [-c, -j, -js].
out_prefix: Prepend this path to the front of all generated files except on
single source targets. Usually is a directory name.
includes: Optional, list of filegroups of schemas that the srcs depend on.
include_paths: Optional, list of paths the includes files can be found in.
flatc_args: Optional, list of additional arguments to pass to flatc.
reflection_name: Optional, if set this will generate the flatbuffer
reflection binaries for the schemas.
reflection_visiblity: The visibility of the generated reflection Fileset.
output_to_bindir: Passed to genrule for output to bin directory.
Outs:
filegroup(name): all generated source files.
Fileset([reflection_name]): (Optional) all generated reflection binaries.
'''
include_paths_cmd = ["-I %s" % (s) for s in include_paths]
# '$(@D)' when given a single source target will give the appropriate
# directory. Appending 'out_prefix' is only necessary when given a build
# target with multiple sources.
output_directory = (
("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)"))
genrule_cmd = " ".join([
"for f in $(SRCS); do",
"$(location %s)" % (flatc_path),
" ".join(flatc_args),
" ".join(include_paths_cmd),
language_flag,
output_directory,
"$$f;",
"done",
])
native.genrule(
name=name,
srcs=srcs,
outs=outs,
output_to_bindir=output_to_bindir,
tools=includes + [flatc_path,],
cmd=genrule_cmd,
message="Generating flatbuffer files for %s:" % (name),)
if reflection_name:
reflection_genrule_cmd = " ".join([
Outs:
filegroup(name): all generated source files.
Fileset([reflection_name]): (Optional) all generated reflection binaries.
Args:
name: Rule name.
srcs: Source .fbs files. Sent in order to the compiler.
outs: Output files from flatc.
language_flag: Target language flag. One of [-c, -j, -js].
out_prefix: Prepend this path to the front of all generated files except on
single source targets. Usually is a directory name.
includes: Optional, list of filegroups of schemas that the srcs depend on.
include_paths: Optional, list of paths the includes files can be found in.
flatc_args: Optional, list of additional arguments to pass to flatc.
reflection_name: Optional, if set this will generate the flatbuffer
reflection binaries for the schemas.
reflection_visiblity: The visibility of the generated reflection Fileset.
output_to_bindir: Passed to genrule for output to bin directory.
"""
include_paths_cmd = ["-I %s" % (s) for s in include_paths]
# '$(@D)' when given a single source target will give the appropriate
# directory. Appending 'out_prefix' is only necessary when given a build
# target with multiple sources.
output_directory = (
("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)")
)
genrule_cmd = " ".join([
"for f in $(SRCS); do",
"$(location %s)" % (flatc_path),
"-b --schema",
" ".join(flatc_args),
" ".join(include_paths_cmd),
language_flag,
@ -75,122 +58,157 @@ def flatbuffer_library_public(name,
"$$f;",
"done",
])
reflection_outs = [
(out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs
]
native.genrule(
name= "%s_srcs" % reflection_name,
srcs=srcs,
outs=reflection_outs,
output_to_bindir=output_to_bindir,
tools=includes + [flatc_path,],
cmd=reflection_genrule_cmd,
message="Generating flatbuffer reflection binary for %s:" % (name),)
native.Fileset(
name=reflection_name,
out="%s_out" % reflection_name,
entries=[
native.FilesetEntry(files=reflection_outs),
name = name,
srcs = srcs,
outs = outs,
output_to_bindir = output_to_bindir,
tools = includes + [flatc_path],
cmd = genrule_cmd,
message = "Generating flatbuffer files for %s:" % (name),
)
if reflection_name:
reflection_genrule_cmd = " ".join([
"for f in $(SRCS); do",
"$(location %s)" % (flatc_path),
"-b --schema",
" ".join(flatc_args),
" ".join(include_paths_cmd),
language_flag,
output_directory,
"$$f;",
"done",
])
reflection_outs = [
(out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1])
for s in srcs
]
native.genrule(
name = "%s_srcs" % reflection_name,
srcs = srcs,
outs = reflection_outs,
output_to_bindir = output_to_bindir,
tools = includes + [flatc_path],
cmd = reflection_genrule_cmd,
message = "Generating flatbuffer reflection binary for %s:" % (name),
)
native.Fileset(
name = reflection_name,
out = "%s_out" % reflection_name,
entries = [
native.FilesetEntry(files = reflection_outs),
],
visibility = reflection_visiblity,
)
def flatbuffer_cc_library(
name,
srcs,
srcs_filegroup_name = "",
out_prefix = "",
includes = [],
include_paths = [],
flatc_args = DEFAULT_FLATC_ARGS,
visibility = None,
srcs_filegroup_visibility = None,
gen_reflections = False):
'''A cc_library with the generated reader/writers for the given flatbuffer definitions.
Outs:
filegroup([name]_srcs): all generated .h files.
filegroup(srcs_filegroup_name if specified, or [name]_includes if not):
Other flatbuffer_cc_library's can pass this in for their `includes`
parameter, if they depend on the schemas in this library.
Fileset([name]_reflection): (Optional) all generated reflection binaries.
cc_library([name]): library with sources and flatbuffers deps.
Remarks:
** Because the genrule used to call flatc does not have any trivial way of
computing the output list of files transitively generated by includes and
--gen-includes (the default) being defined for flatc, the --gen-includes
flag will not work as expected. The way around this is to add a dependency
to the flatbuffer_cc_library defined alongside the flatc included Fileset.
For example you might define:
flatbuffer_cc_library(
name = "my_fbs",
srcs = [ "schemas/foo.fbs" ],
includes = [ "//third_party/bazz:bazz_fbs_includes" ],
)
In which foo.fbs includes a few files from the Fileset defined at
//third_party/bazz:bazz_fbs_includes. When compiling the library that
includes foo_generated.h, and therefore has my_fbs as a dependency, it
will fail to find any of the bazz *_generated.h files unless you also
add bazz's flatbuffer_cc_library to your own dependency list, e.g.:
cc_library(
name = "my_lib",
deps = [
":my_fbs",
"//third_party/bazz:bazz_fbs"
],
)
Happy dependent Flatbuffering!
Args:
name: Rule name.
srcs: Source .fbs files. Sent in order to the compiler.
srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this
filegroup into the `includes` parameter of any other
flatbuffer_cc_library that depends on this one's schemas.
out_prefix: Prepend this path to the front of all generated files. Usually
is a directory name.
includes: Optional, list of filegroups of schemas that the srcs depend on.
** SEE REMARKS BELOW **
include_paths: Optional, list of paths the includes files can be found in.
flatc_args: Optional list of additional arguments to pass to flatc
(e.g. --gen-mutable).
visibility: The visibility of the generated cc_library. By default, use the
default visibility of the project.
srcs_filegroup_visibility: The visibility of the generated srcs filegroup.
By default, use the value of the visibility parameter above.
gen_reflections: Optional, if true this will generate the flatbuffer
reflection binaries for the schemas.
'''
output_headers = [
(out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1])
for s in srcs
]
reflection_name = "%s_reflection" % name if gen_reflections else ""
flatbuffer_library_public(
name = "%s_srcs" % (name),
srcs = srcs,
outs = output_headers,
language_flag = "-c",
out_prefix = out_prefix,
includes = includes,
include_paths = include_paths,
flatc_args = flatc_args,
reflection_name = reflection_name,
reflection_visiblity = visibility,
)
native.cc_library(
name = name,
hdrs = output_headers,
srcs = output_headers,
features = [
"-parse_headers",
],
visibility=reflection_visiblity
deps = [
"@flatbuffers//:runtime_cc",
],
includes = ["."],
linkstatic = 1,
visibility = visibility,
)
def flatbuffer_cc_library(name, srcs, srcs_filegroup_name="",
out_prefix="", includes=[], include_paths=[],
flatc_args=DEFAULT_FLATC_ARGS,
visibility=None, srcs_filegroup_visibility=None,
gen_reflections=False):
'''A cc_library with the generated reader/writers for the given flatbuffer definitions.
Args:
name: Rule name.
srcs: Source .fbs files. Sent in order to the compiler.
srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this
filegroup into the `includes` parameter of any other
flatbuffer_cc_library that depends on this one's schemas.
out_prefix: Prepend this path to the front of all generated files. Usually
is a directory name.
includes: Optional, list of filegroups of schemas that the srcs depend on.
** SEE REMARKS BELOW **
include_paths: Optional, list of paths the includes files can be found in.
flatc_args: Optional list of additional arguments to pass to flatc
(e.g. --gen-mutable).
visibility: The visibility of the generated cc_library. By default, use the
default visibility of the project.
srcs_filegroup_visibility: The visibility of the generated srcs filegroup.
By default, use the value of the visibility parameter above.
gen_reflections: Optional, if true this will generate the flatbuffer
reflection binaries for the schemas.
Outs:
filegroup([name]_srcs): all generated .h files.
filegroup(srcs_filegroup_name if specified, or [name]_includes if not):
Other flatbuffer_cc_library's can pass this in for their `includes`
parameter, if they depend on the schemas in this library.
Fileset([name]_reflection): (Optional) all generated reflection binaries.
cc_library([name]): library with sources and flatbuffers deps.
Remarks:
** Because the genrule used to call flatc does not have any trivial way of
computing the output list of files transitively generated by includes and
--gen-includes (the default) being defined for flatc, the --gen-includes
flag will not work as expected. The way around this is to add a dependency
to the flatbuffer_cc_library defined alongside the flatc included Fileset.
For example you might define:
flatbuffer_cc_library(
name = "my_fbs",
srcs = [ "schemas/foo.fbs" ],
includes = [ "//third_party/bazz:bazz_fbs_includes" ],
)
In which foo.fbs includes a few files from the Fileset defined at
//third_party/bazz:bazz_fbs_includes. When compiling the library that
includes foo_generated.h, and therefore has my_fbs as a dependency, it
will fail to find any of the bazz *_generated.h files unless you also
add bazz's flatbuffer_cc_library to your own dependency list, e.g.:
cc_library(
name = "my_lib",
deps = [
":my_fbs",
"//third_party/bazz:bazz_fbs"
],
)
Happy dependent Flatbuffering!
'''
output_headers = [
(out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs
]
reflection_name = "%s_reflection" % name if gen_reflections else ""
flatbuffer_library_public(name="%s_srcs" % (name),
srcs=srcs,
outs=output_headers,
language_flag="-c",
out_prefix=out_prefix,
includes=includes,
include_paths=include_paths,
flatc_args=flatc_args,
reflection_name=reflection_name,
reflection_visiblity=visibility,)
native.cc_library(name=name,
hdrs=output_headers,
srcs=output_headers,
features=[
"-parse_headers",
],
deps=[
"@flatbuffers//:runtime_cc",
],
includes=["."],
linkstatic=1,
visibility=visibility)
# A filegroup for the `srcs`. That is, all the schema files for this
# Flatbuffer set.
native.filegroup(
name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name),
srcs = srcs,
visibility=srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility)
# A filegroup for the `srcs`. That is, all the schema files for this
# Flatbuffer set.
native.filegroup(
name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name),
srcs = srcs,
visibility = srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility,
)

19
third_party/flatbuffers/workspace.bzl vendored Normal file
View File

@ -0,0 +1,19 @@
"""Loads the Flatbuffers library, used by TF Lite."""
load("//third_party:repo.bzl", "third_party_http_archive")
def repo():
third_party_http_archive(
name = "flatbuffers",
strip_prefix = "flatbuffers-1.9.0",
sha256 = "5ca5491e4260cacae30f1a5786d109230db3f3a6e5a0eb45d0d0608293d247e3",
urls = [
"https://mirror.bazel.build/github.com/google/flatbuffers/archive/v1.9.0.tar.gz",
"https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz",
],
build_file = "//third_party/flatbuffers:BUILD.bazel",
system_build_file = "//third_party/flatbuffers:BUILD.system",
link_files = {
"//third_party/flatbuffers:build_defs.bzl": "build_defs.bzl",
},
)