Convert TensorFlow's flatbuffers dependency to new third party import method

PiperOrigin-RevId: 209833197

tensorflow/contrib/lite/delegates/eager/kernel.cc

@@ -14,7 +14,7 @@ limitations under the License.
 ==============================================================================*/
 #include "tensorflow/contrib/lite/delegates/eager/kernel.h"
 
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/builtin_ops.h"
 #include "tensorflow/contrib/lite/context.h"
 #include "tensorflow/contrib/lite/context_util.h"

tensorflow/contrib/lite/delegates/eager/test_util.cc

@@ -16,7 +16,7 @@ limitations under the License.
 #include "tensorflow/contrib/lite/delegates/eager/test_util.h"
 
 #include "absl/memory/memory.h"
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/string.h"
 
 namespace tflite {

tensorflow/contrib/lite/experimental/kernels/ctc_beam_search_decoder.cc

@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include <vector>
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/context.h"
 #include "tensorflow/contrib/lite/experimental/kernels/ctc_beam_search.h"
 #include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h"

tensorflow/contrib/lite/experimental/kernels/ctc_beam_search_decoder_test.cc

@@ -18,7 +18,7 @@ limitations under the License.
 #include <vector>
 
 #include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/interpreter.h"
 #include "tensorflow/contrib/lite/kernels/register.h"
 #include "tensorflow/contrib/lite/kernels/test_util.h"

tensorflow/contrib/lite/kernels/audio_spectrogram.cc

@@ -22,7 +22,7 @@ limitations under the License.
 #include "tensorflow/contrib/lite/kernels/kernel_util.h"
 #include "tensorflow/contrib/lite/kernels/op_macros.h"
 
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 
 namespace tflite {
 namespace ops {

tensorflow/contrib/lite/kernels/audio_spectrogram_test.cc

@@ -18,7 +18,7 @@ limitations under the License.
 #include <vector>
 
 #include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/interpreter.h"
 #include "tensorflow/contrib/lite/kernels/register.h"
 #include "tensorflow/contrib/lite/kernels/test_util.h"

tensorflow/contrib/lite/kernels/detection_postprocess.cc

@@ -15,7 +15,7 @@ limitations under the License.
 #include <string.h>
 #include <numeric>
 #include <vector>
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/builtin_op_data.h"
 #include "tensorflow/contrib/lite/context.h"
 #include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h"

tensorflow/contrib/lite/kernels/detection_postprocess_test.cc

@@ -17,7 +17,7 @@ limitations under the License.
 #include <vector>
 
 #include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/interpreter.h"
 #include "tensorflow/contrib/lite/kernels/register.h"
 #include "tensorflow/contrib/lite/kernels/test_util.h"

tensorflow/contrib/lite/kernels/mfcc.cc

@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 #include "tensorflow/contrib/lite/kernels/internal/mfcc.h"
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/builtin_op_data.h"
 #include "tensorflow/contrib/lite/context.h"
 #include "tensorflow/contrib/lite/kernels/internal/mfcc_dct.h"

tensorflow/contrib/lite/kernels/mfcc_test.cc

@@ -18,7 +18,7 @@ limitations under the License.
 #include <vector>
 
 #include <gtest/gtest.h>
-#include "flatbuffers/flexbuffers.h"
+#include "flatbuffers/flexbuffers.h"  // flatbuffers
 #include "tensorflow/contrib/lite/interpreter.h"
 #include "tensorflow/contrib/lite/kernels/register.h"
 #include "tensorflow/contrib/lite/kernels/test_util.h"

tensorflow/contrib/lite/schema/BUILD

@@ -48,7 +48,7 @@ exports_files([
     "schema_v3.fbs",
 ])
 
-load("//third_party/flatbuffers:build_defs.bzl", "flatbuffer_cc_library")
+load("@flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
 
 # Generic schema for inference on device.
 flatbuffer_cc_library(

tensorflow/contrib/lite/schema/flatbuffer_compatibility_test.cc

@@ -15,7 +15,7 @@ limitations under the License.
 
 #include <fstream>
 #include <gtest/gtest.h>
-#include "flatbuffers/flatc.h"
+#include "flatbuffers/flatc.h"  // flatbuffers
 #include "tensorflow/core/platform/platform.h"
 
 #ifdef PLATFORM_GOOGLE

tensorflow/workspace.bzl

@@ -19,10 +19,10 @@ load(
     "//tensorflow/tools/def_file_filter:def_file_filter_configure.bzl",
     "def_file_filter_configure",
 )
+load("//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo")
 
 def initialize_third_party():
-    # Fill in later
+    flatbuffers()
-    pass
 
 # Sanitize a dependency so that it works correctly from code that includes
 # TensorFlow as a submodule.
@@ -740,18 +740,6 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""):
         build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"),
     )
 
-    tf_http_archive(
-        name = "flatbuffers",
-        strip_prefix = "flatbuffers-1.9.0",
-        sha256 = "5ca5491e4260cacae30f1a5786d109230db3f3a6e5a0eb45d0d0608293d247e3",
-        urls = [
-            "https://mirror.bazel.build/github.com/google/flatbuffers/archive/v1.9.0.tar.gz",
-            "https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz",
-        ],
-        build_file = clean_dep("//third_party/flatbuffers:flatbuffers.BUILD"),
-        system_build_file = clean_dep("//third_party/systemlibs:flatbuffers.BUILD"),
-    )
-
     native.new_http_archive(
         name = "double_conversion",
         urls = [

third_party/flatbuffers/BUILD

@@ -1,15 +1 @@
-package(default_visibility = ["//visibility:public"])
+# This empty BUILD file is required to make Bazel treat this directory as a package.
-
-licenses(["notice"])  # Apache 2.0
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)

third_party/flatbuffers/BUILD.bazel

@@ -19,7 +19,10 @@ config_setting(
 
 FLATBUFFERS_COPTS = select({
     ":windows": [],
-    "//conditions:default": ["-Wno-implicit-fallthrough", "-fexceptions"],
+    "//conditions:default": [
+        "-Wno-implicit-fallthrough",
+        "-fexceptions",
+    ],
 })
 
 # Public flatc library to compile flatbuffer files at runtime.

third_party/flatbuffers/build_defs.bzl

@@ -1,5 +1,4 @@
-# Description:
+"""BUILD rules for generating flatbuffer files."""
-#   BUILD rules for generating flatbuffer files.
 
 flatc_path = "@flatbuffers//:flatc"
 
@@ -8,66 +7,50 @@ DEFAULT_FLATC_ARGS = [
     "--gen-object-api",
 ]
 
-def flatbuffer_library_public(name,
+def flatbuffer_library_public(
-                              srcs,
+        name,
-                              outs,
+        srcs,
-                              language_flag,
+        outs,
-                              out_prefix="",
+        language_flag,
-                              includes=[],
+        out_prefix = "",
-                              include_paths=[],
+        includes = [],
-                              flatc_args=DEFAULT_FLATC_ARGS,
+        include_paths = [],
-                              reflection_name="",
+        flatc_args = DEFAULT_FLATC_ARGS,
-                              reflection_visiblity=None,
+        reflection_name = "",
-                              output_to_bindir=False):
+        reflection_visiblity = None,
-  '''Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler.
+        output_to_bindir = False):
+    """Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler.
 
-  Args:
+    Outs:
-    name: Rule name.
+      filegroup(name): all generated source files.
-    srcs: Source .fbs files. Sent in order to the compiler.
+      Fileset([reflection_name]): (Optional) all generated reflection binaries.
-    outs: Output files from flatc.
+
-    language_flag: Target language flag. One of [-c, -j, -js].
+    Args:
-    out_prefix: Prepend this path to the front of all generated files except on
+      name: Rule name.
-        single source targets. Usually is a directory name.
+      srcs: Source .fbs files. Sent in order to the compiler.
-    includes: Optional, list of filegroups of schemas that the srcs depend on.
+      outs: Output files from flatc.
-    include_paths: Optional, list of paths the includes files can be found in.
+      language_flag: Target language flag. One of [-c, -j, -js].
-    flatc_args: Optional, list of additional arguments to pass to flatc.
+      out_prefix: Prepend this path to the front of all generated files except on
-    reflection_name: Optional, if set this will generate the flatbuffer
+          single source targets. Usually is a directory name.
-      reflection binaries for the schemas.
+      includes: Optional, list of filegroups of schemas that the srcs depend on.
-    reflection_visiblity: The visibility of the generated reflection Fileset.
+      include_paths: Optional, list of paths the includes files can be found in.
-    output_to_bindir: Passed to genrule for output to bin directory.
+      flatc_args: Optional, list of additional arguments to pass to flatc.
-  Outs:
+      reflection_name: Optional, if set this will generate the flatbuffer
-    filegroup(name): all generated source files.
+        reflection binaries for the schemas.
-    Fileset([reflection_name]): (Optional) all generated reflection binaries.
+      reflection_visiblity: The visibility of the generated reflection Fileset.
-  '''
+      output_to_bindir: Passed to genrule for output to bin directory.
-  include_paths_cmd = ["-I %s" % (s) for s in include_paths]
+    """
-  # '$(@D)' when given a single source target will give the appropriate
+    include_paths_cmd = ["-I %s" % (s) for s in include_paths]
-  # directory. Appending 'out_prefix' is only necessary when given a build
+
-  # target with multiple sources.
+    # '$(@D)' when given a single source target will give the appropriate
-  output_directory = (
+    # directory. Appending 'out_prefix' is only necessary when given a build
-      ("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)"))
+    # target with multiple sources.
-  genrule_cmd = " ".join([
+    output_directory = (
-      "for f in $(SRCS); do",
+        ("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)")
-      "$(location %s)" % (flatc_path),
+    )
-      " ".join(flatc_args),
+    genrule_cmd = " ".join([
-      " ".join(include_paths_cmd),
-      language_flag,
-      output_directory,
-      "$$f;",
-      "done",
-  ])
-  native.genrule(
-      name=name,
-      srcs=srcs,
-      outs=outs,
-      output_to_bindir=output_to_bindir,
-      tools=includes + [flatc_path,],
-      cmd=genrule_cmd,
-      message="Generating flatbuffer files for %s:" % (name),)
-  if reflection_name:
-    reflection_genrule_cmd = " ".join([
         "for f in $(SRCS); do",
         "$(location %s)" % (flatc_path),
-        "-b --schema",
         " ".join(flatc_args),
         " ".join(include_paths_cmd),
         language_flag,
@@ -75,122 +58,157 @@ def flatbuffer_library_public(name,
         "$$f;",
         "done",
     ])
-    reflection_outs = [
-        (out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs
-    ]
     native.genrule(
-        name= "%s_srcs" % reflection_name,
+        name = name,
-        srcs=srcs,
+        srcs = srcs,
-        outs=reflection_outs,
+        outs = outs,
-        output_to_bindir=output_to_bindir,
+        output_to_bindir = output_to_bindir,
-        tools=includes + [flatc_path,],
+        tools = includes + [flatc_path],
-        cmd=reflection_genrule_cmd,
+        cmd = genrule_cmd,
-        message="Generating flatbuffer reflection binary for %s:" % (name),)
+        message = "Generating flatbuffer files for %s:" % (name),
-    native.Fileset(
+    )
-        name=reflection_name,
+    if reflection_name:
-        out="%s_out" % reflection_name,
+        reflection_genrule_cmd = " ".join([
-        entries=[
+            "for f in $(SRCS); do",
-            native.FilesetEntry(files=reflection_outs),
+            "$(location %s)" % (flatc_path),
+            "-b --schema",
+            " ".join(flatc_args),
+            " ".join(include_paths_cmd),
+            language_flag,
+            output_directory,
+            "$$f;",
+            "done",
+        ])
+        reflection_outs = [
+            (out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1])
+            for s in srcs
+        ]
+        native.genrule(
+            name = "%s_srcs" % reflection_name,
+            srcs = srcs,
+            outs = reflection_outs,
+            output_to_bindir = output_to_bindir,
+            tools = includes + [flatc_path],
+            cmd = reflection_genrule_cmd,
+            message = "Generating flatbuffer reflection binary for %s:" % (name),
+        )
+        native.Fileset(
+            name = reflection_name,
+            out = "%s_out" % reflection_name,
+            entries = [
+                native.FilesetEntry(files = reflection_outs),
+            ],
+            visibility = reflection_visiblity,
+        )
+
+def flatbuffer_cc_library(
+        name,
+        srcs,
+        srcs_filegroup_name = "",
+        out_prefix = "",
+        includes = [],
+        include_paths = [],
+        flatc_args = DEFAULT_FLATC_ARGS,
+        visibility = None,
+        srcs_filegroup_visibility = None,
+        gen_reflections = False):
+    '''A cc_library with the generated reader/writers for the given flatbuffer definitions.
+
+    Outs:
+      filegroup([name]_srcs): all generated .h files.
+      filegroup(srcs_filegroup_name if specified, or [name]_includes if not):
+          Other flatbuffer_cc_library's can pass this in for their `includes`
+          parameter, if they depend on the schemas in this library.
+      Fileset([name]_reflection): (Optional) all generated reflection binaries.
+      cc_library([name]): library with sources and flatbuffers deps.
+
+    Remarks:
+      ** Because the genrule used to call flatc does not have any trivial way of
+        computing the output list of files transitively generated by includes and
+        --gen-includes (the default) being defined for flatc, the --gen-includes
+        flag will not work as expected. The way around this is to add a dependency
+        to the flatbuffer_cc_library defined alongside the flatc included Fileset.
+        For example you might define:
+
+        flatbuffer_cc_library(
+            name = "my_fbs",
+            srcs = [ "schemas/foo.fbs" ],
+            includes = [ "//third_party/bazz:bazz_fbs_includes" ],
+        )
+
+        In which foo.fbs includes a few files from the Fileset defined at
+        //third_party/bazz:bazz_fbs_includes. When compiling the library that
+        includes foo_generated.h, and therefore has my_fbs as a dependency, it
+        will fail to find any of the bazz *_generated.h files unless you also
+        add bazz's flatbuffer_cc_library to your own dependency list, e.g.:
+
+        cc_library(
+            name = "my_lib",
+            deps = [
+                ":my_fbs",
+                "//third_party/bazz:bazz_fbs"
+            ],
+        )
+
+        Happy dependent Flatbuffering!
+
+    Args:
+      name: Rule name.
+      srcs: Source .fbs files. Sent in order to the compiler.
+      srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this
+          filegroup into the `includes` parameter of any other
+          flatbuffer_cc_library that depends on this one's schemas.
+      out_prefix: Prepend this path to the front of all generated files. Usually
+          is a directory name.
+      includes: Optional, list of filegroups of schemas that the srcs depend on.
+          ** SEE REMARKS BELOW **
+      include_paths: Optional, list of paths the includes files can be found in.
+      flatc_args: Optional list of additional arguments to pass to flatc
+          (e.g. --gen-mutable).
+      visibility: The visibility of the generated cc_library. By default, use the
+          default visibility of the project.
+      srcs_filegroup_visibility: The visibility of the generated srcs filegroup.
+          By default, use the value of the visibility parameter above.
+      gen_reflections: Optional, if true this will generate the flatbuffer
+        reflection binaries for the schemas.
+    '''
+    output_headers = [
+        (out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1])
+        for s in srcs
+    ]
+    reflection_name = "%s_reflection" % name if gen_reflections else ""
+
+    flatbuffer_library_public(
+        name = "%s_srcs" % (name),
+        srcs = srcs,
+        outs = output_headers,
+        language_flag = "-c",
+        out_prefix = out_prefix,
+        includes = includes,
+        include_paths = include_paths,
+        flatc_args = flatc_args,
+        reflection_name = reflection_name,
+        reflection_visiblity = visibility,
+    )
+    native.cc_library(
+        name = name,
+        hdrs = output_headers,
+        srcs = output_headers,
+        features = [
+            "-parse_headers",
         ],
-        visibility=reflection_visiblity
+        deps = [
+            "@flatbuffers//:runtime_cc",
+        ],
+        includes = ["."],
+        linkstatic = 1,
+        visibility = visibility,
     )
 
-
+    # A filegroup for the `srcs`. That is, all the schema files for this
-def flatbuffer_cc_library(name, srcs, srcs_filegroup_name="",
+    # Flatbuffer set.
-                          out_prefix="", includes=[], include_paths=[],
+    native.filegroup(
-                          flatc_args=DEFAULT_FLATC_ARGS,
+        name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name),
-                          visibility=None, srcs_filegroup_visibility=None,
+        srcs = srcs,
-                          gen_reflections=False):
+        visibility = srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility,
-  '''A cc_library with the generated reader/writers for the given flatbuffer definitions.
+    )
-
-  Args:
-    name: Rule name.
-    srcs: Source .fbs files. Sent in order to the compiler.
-    srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this
-        filegroup into the `includes` parameter of any other
-        flatbuffer_cc_library that depends on this one's schemas.
-    out_prefix: Prepend this path to the front of all generated files. Usually
-        is a directory name.
-    includes: Optional, list of filegroups of schemas that the srcs depend on.
-        ** SEE REMARKS BELOW **
-    include_paths: Optional, list of paths the includes files can be found in.
-    flatc_args: Optional list of additional arguments to pass to flatc
-        (e.g. --gen-mutable).
-    visibility: The visibility of the generated cc_library. By default, use the
-        default visibility of the project.
-    srcs_filegroup_visibility: The visibility of the generated srcs filegroup.
-        By default, use the value of the visibility parameter above.
-    gen_reflections: Optional, if true this will generate the flatbuffer
-      reflection binaries for the schemas.
-  Outs:
-    filegroup([name]_srcs): all generated .h files.
-    filegroup(srcs_filegroup_name if specified, or [name]_includes if not):
-        Other flatbuffer_cc_library's can pass this in for their `includes`
-        parameter, if they depend on the schemas in this library.
-    Fileset([name]_reflection): (Optional) all generated reflection binaries.
-    cc_library([name]): library with sources and flatbuffers deps.
-
-  Remarks:
-    ** Because the genrule used to call flatc does not have any trivial way of
-      computing the output list of files transitively generated by includes and
-      --gen-includes (the default) being defined for flatc, the --gen-includes
-      flag will not work as expected. The way around this is to add a dependency
-      to the flatbuffer_cc_library defined alongside the flatc included Fileset.
-      For example you might define:
-
-      flatbuffer_cc_library(
-          name = "my_fbs",
-          srcs = [ "schemas/foo.fbs" ],
-          includes = [ "//third_party/bazz:bazz_fbs_includes" ],
-      )
-
-      In which foo.fbs includes a few files from the Fileset defined at
-      //third_party/bazz:bazz_fbs_includes. When compiling the library that
-      includes foo_generated.h, and therefore has my_fbs as a dependency, it
-      will fail to find any of the bazz *_generated.h files unless you also
-      add bazz's flatbuffer_cc_library to your own dependency list, e.g.:
-
-      cc_library(
-          name = "my_lib",
-          deps = [
-              ":my_fbs",
-              "//third_party/bazz:bazz_fbs"
-          ],
-      )
-
-      Happy dependent Flatbuffering!
-  '''
-  output_headers = [
-      (out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs
-  ]
-  reflection_name = "%s_reflection" % name if gen_reflections else ""
-
-  flatbuffer_library_public(name="%s_srcs" % (name),
-                            srcs=srcs,
-                            outs=output_headers,
-                            language_flag="-c",
-                            out_prefix=out_prefix,
-                            includes=includes,
-                            include_paths=include_paths,
-                            flatc_args=flatc_args,
-                            reflection_name=reflection_name,
-                            reflection_visiblity=visibility,)
-  native.cc_library(name=name,
-                    hdrs=output_headers,
-                    srcs=output_headers,
-                    features=[
-                        "-parse_headers",
-                    ],
-                    deps=[
-                        "@flatbuffers//:runtime_cc",
-                    ],
-                    includes=["."],
-                    linkstatic=1,
-                    visibility=visibility)
-
-  # A filegroup for the `srcs`. That is, all the schema files for this
-  # Flatbuffer set.
-  native.filegroup(
-      name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name),
-      srcs = srcs,
-      visibility=srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility)

third_party/flatbuffers/workspace.bzl

@@ -0,0 +1,19 @@
+"""Loads the Flatbuffers library, used by TF Lite."""
+
+load("//third_party:repo.bzl", "third_party_http_archive")
+
+def repo():
+    third_party_http_archive(
+        name = "flatbuffers",
+        strip_prefix = "flatbuffers-1.9.0",
+        sha256 = "5ca5491e4260cacae30f1a5786d109230db3f3a6e5a0eb45d0d0608293d247e3",
+        urls = [
+            "https://mirror.bazel.build/github.com/google/flatbuffers/archive/v1.9.0.tar.gz",
+            "https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz",
+        ],
+        build_file = "//third_party/flatbuffers:BUILD.bazel",
+        system_build_file = "//third_party/flatbuffers:BUILD.system",
+        link_files = {
+            "//third_party/flatbuffers:build_defs.bzl": "build_defs.bzl",
+        },
+    )