From 14ac2235699509f512b44b71160239c153ab413d Mon Sep 17 00:00:00 2001 From: "David Z. Chen" Date: Thu, 26 May 2016 15:20:44 -0700 Subject: [PATCH] Use external repository for protobuf dependency. (#1289) Currently, TensorFlow uses a Git submodule for its dependency on protobuf. This was due to the hack used in protobuf for building Python support with Bazel, which was required since Bazel's Python rules did not support adding directories to `PYTHONPATH`. Now that the new `imports` attribute has been added to the Python rules in Bazel 0.2 and the hack for Python support in protobuf has been removed, this change removes the `google/protobuf` Git submodule and adds an external repository for including protobuf. This patch also adds gmock.BUILD to the tensorflow repo. Fixes #1069 Fixes #2021 --- .gitmodules | 3 - WORKSPACE | 2 +- configure | 11 --- gmock.BUILD | 28 +++++++ google/protobuf | 1 - tensorflow/contrib/ffmpeg/default/BUILD | 2 +- tensorflow/contrib/layers/kernels/BUILD | 4 +- .../contrib/linear_optimizer/kernels/BUILD | 6 +- tensorflow/contrib/metrics/kernels/BUILD | 2 +- tensorflow/contrib/tensor_forest/BUILD | 2 +- .../core/platform/default/build_config.bzl | 74 +++++++++++-------- tensorflow/python/BUILD | 4 +- tensorflow/tensorflow.bzl | 2 +- tensorflow/tools/benchmark/BUILD | 2 +- tensorflow/tools/pip_package/BUILD | 2 +- .../tools/pip_package/build_pip_package.sh | 3 +- tensorflow/workspace.bzl | 49 +++++++----- 17 files changed, 118 insertions(+), 79 deletions(-) create mode 100644 gmock.BUILD delete mode 160000 google/protobuf diff --git a/.gitmodules b/.gitmodules index 1b17ea57b9d..e69de29bb2d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "google/protobuf"] - path = google/protobuf - url = https://github.com/google/protobuf.git diff --git a/WORKSPACE b/WORKSPACE index ffebcde5541..04d0e0d3fa1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -20,7 +20,7 @@ tf_workspace() # Specify the minimum required bazel version. load("//tensorflow:tensorflow.bzl", "check_version") -check_version("0.1.4") +check_version("0.2.0") # TENSORBOARD_BOWER_AUTOGENERATED_BELOW_THIS_LINE_DO_NOT_EDIT diff --git a/configure b/configure index 5d6397da57d..98048ba91db 100755 --- a/configure +++ b/configure @@ -2,17 +2,6 @@ DO_NOT_SUBMIT_WARNING="Unofficial setting. DO NOT SUBMIT!!!" -## Verify that the submodule google/protobuf is available -# TODO(cais): Remove this check once protobuf is no longer depended upon -if [[ ! -f "google/protobuf/protobuf.bzl" ]]; then - echo "ERROR: It appears that the required submodule google/protobuf is not "\ -"available in this TensorFlow git clone." - echo "Please be sure to use the --recurse-submodules flag when performing "\ -"git clone of TensorFlow." - - exit 1 -fi - ## Set up python-related environment settings while true; do fromuser="" diff --git a/gmock.BUILD b/gmock.BUILD new file mode 100644 index 00000000000..82abf275408 --- /dev/null +++ b/gmock.BUILD @@ -0,0 +1,28 @@ +cc_library( + name = "gtest", + srcs = [ + "gmock-1.7.0/gtest/src/gtest-all.cc", + "gmock-1.7.0/src/gmock-all.cc", + ], + hdrs = glob([ + "gmock-1.7.0/**/*.h", + "gmock-1.7.0/gtest/src/*.cc", + "gmock-1.7.0/src/*.cc", + ]), + includes = [ + "gmock-1.7.0", + "gmock-1.7.0/gtest", + "gmock-1.7.0/gtest/include", + "gmock-1.7.0/include", + ], + linkopts = ["-pthread"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "gtest_main", + srcs = ["gmock-1.7.0/src/gmock_main.cc"], + linkopts = ["-pthread"], + visibility = ["//visibility:public"], + deps = [":gtest"], +) diff --git a/google/protobuf b/google/protobuf deleted file mode 160000 index fb714b3606b..00000000000 --- a/google/protobuf +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fb714b3606bd663b823f6960a73d052f97283b74 diff --git a/tensorflow/contrib/ffmpeg/default/BUILD b/tensorflow/contrib/ffmpeg/default/BUILD index e1b7bb61924..6e25a61e430 100644 --- a/tensorflow/contrib/ffmpeg/default/BUILD +++ b/tensorflow/contrib/ffmpeg/default/BUILD @@ -16,7 +16,7 @@ cc_library( "//tensorflow/contrib/ffmpeg:ffmpeg_lib.h", ], deps = [ - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", ], ) diff --git a/tensorflow/contrib/layers/kernels/BUILD b/tensorflow/contrib/layers/kernels/BUILD index 62926e2df41..c540147cd89 100644 --- a/tensorflow/contrib/layers/kernels/BUILD +++ b/tensorflow/contrib/layers/kernels/BUILD @@ -11,7 +11,7 @@ cc_library( name = "bucketization_kernel", srcs = ["bucketization_kernel.cc"], deps = [ - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", "//third_party/eigen3", ], @@ -22,7 +22,7 @@ cc_library( name = "sparse_feature_cross_kernel", srcs = ["sparse_feature_cross_kernel.cc"], deps = [ - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", "//third_party/eigen3", ], diff --git a/tensorflow/contrib/linear_optimizer/kernels/BUILD b/tensorflow/contrib/linear_optimizer/kernels/BUILD index ac9a33ccc4d..95a9e4b6ac4 100644 --- a/tensorflow/contrib/linear_optimizer/kernels/BUILD +++ b/tensorflow/contrib/linear_optimizer/kernels/BUILD @@ -21,7 +21,7 @@ cc_library( "squared-loss.h", ], deps = [ - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", ], ) @@ -43,7 +43,7 @@ cc_library( srcs = ["resources.cc"], hdrs = ["resources.h"], deps = [ - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", "//third_party/eigen3", ], @@ -68,7 +68,7 @@ cc_library( deps = [ ":loss_updaters", ":resources", - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", "//tensorflow/core/kernels:bounds_check_lib", "//third_party/eigen3", diff --git a/tensorflow/contrib/metrics/kernels/BUILD b/tensorflow/contrib/metrics/kernels/BUILD index 4ba121db83a..71dac11c80f 100644 --- a/tensorflow/contrib/metrics/kernels/BUILD +++ b/tensorflow/contrib/metrics/kernels/BUILD @@ -11,7 +11,7 @@ cc_library( name = "set_kernels", srcs = ["set_kernels.cc"], deps = [ - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", "//third_party/eigen3", ], diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD index 33c66a6bc2c..c9934e50ac8 100644 --- a/tensorflow/contrib/tensor_forest/BUILD +++ b/tensorflow/contrib/tensor_forest/BUILD @@ -25,7 +25,7 @@ cc_library( "core/ops/tree_utils.h", ], deps = [ - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", "//third_party/eigen3", ], diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index c20db730795..f6eb40f600a 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -1,7 +1,7 @@ # Platform-specific build configurations. -load("//google/protobuf:protobuf.bzl", "cc_proto_library") -load("//google/protobuf:protobuf.bzl", "py_proto_library") +load("@protobuf//:protobuf.bzl", "cc_proto_library") +load("@protobuf//:protobuf.bzl", "py_proto_library") # configure may change the following line to True WITH_GCP_SUPPORT = False @@ -31,29 +31,39 @@ def tf_proto_library_cc(name, srcs = [], has_services = None, cc_api_version = 2, go_api_version = 2, java_api_version = 2, py_api_version = 2): - native.filegroup(name=name + "_proto_srcs", - srcs=srcs + tf_deps(deps, "_proto_srcs"), - testonly=testonly,) + native.filegroup( + name = name + "_proto_srcs", + srcs = srcs + tf_deps(deps, "_proto_srcs"), + testonly = testonly, + ) use_grpc_plugin = None if cc_grpc_version: use_grpc_plugin = True - cc_proto_library(name=name + "_cc", - srcs=srcs + tf_deps(deps, "_proto_srcs"), - deps=deps + ["//google/protobuf:cc_wkt_protos"], - cc_libs = cc_libs + ["//google/protobuf:protobuf"], - use_grpc_plugin = use_grpc_plugin, - testonly=testonly, - visibility=visibility,) + cc_proto_library( + name = name + "_cc", + srcs = srcs + tf_deps(deps, "_proto_srcs"), + deps = deps + ["@protobuf//:cc_wkt_protos"], + cc_libs = cc_libs + ["@protobuf//:protobuf"], + protoc = "@protobuf//:protoc", + default_runtime = "@protobuf//:protobuf", + use_grpc_plugin = use_grpc_plugin, + testonly = testonly, + visibility = visibility, + ) def tf_proto_library_py(name, srcs=[], deps=[], visibility=[], testonly=0, srcs_version="PY2AND3"): - py_proto_library(name = name + "_py", - srcs = srcs, - srcs_version = srcs_version, - deps = deps, - visibility = visibility, - testonly = testonly) + py_proto_library( + name = name + "_py", + srcs = srcs, + srcs_version = srcs_version, + deps = deps, + protoc = "@protobuf//:protoc", + default_runtime = "@protobuf//:protobuf_python", + visibility = visibility, + testonly = testonly, + ) def tf_proto_library(name, srcs = [], has_services = None, deps = [], visibility = [], testonly = 0, @@ -61,19 +71,23 @@ def tf_proto_library(name, srcs = [], has_services = None, cc_api_version = 2, go_api_version = 2, java_api_version = 2, py_api_version = 2): - tf_proto_library_cc(name=name, - srcs=srcs + tf_deps(deps, "_proto_srcs"), - deps=deps, - cc_libs=cc_libs, - testonly=testonly, - visibility=visibility,) + tf_proto_library_cc( + name = name, + srcs = srcs + tf_deps(deps, "_proto_srcs"), + deps = deps, + cc_libs = cc_libs, + testonly = testonly, + visibility = visibility, + ) - tf_proto_library_py(name=name, - srcs=srcs + tf_deps(deps, "_proto_srcs"), - srcs_version="PY2AND3", - deps=deps + ["//google/protobuf:protobuf_python"], - testonly=testonly, - visibility=visibility,) + tf_proto_library_py( + name = name, + srcs = srcs + tf_deps(deps, "_proto_srcs"), + srcs_version = "PY2AND3", + deps = deps + ["@protobuf//:protobuf_python"], + testonly = testonly, + visibility = visibility, + ) def tf_additional_lib_srcs(): return [ diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 4b290f15097..b7e2a462e57 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -134,7 +134,7 @@ cc_binary( }), linkshared = 1, deps = [ - "//google/protobuf", + "@protobuf//:protobuf", "//tensorflow/core:framework_headers_lib", ], ) @@ -885,7 +885,7 @@ py_library( name = "util", srcs = glob(["util/**/*.py"]), srcs_version = "PY2AND3", - deps = ["//google/protobuf:protobuf_python"], + deps = ["@protobuf//:protobuf_python"], ) tf_proto_library( diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 4eb5619ecd7..bc22150f237 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -539,7 +539,7 @@ def cc_header_only_library(name, deps=[], **kwargs): def tf_custom_op_library_additional_deps(): return [ - "//google/protobuf", + "@protobuf//:protobuf", "//third_party/eigen3", "//tensorflow/core:framework_headers_lib", ] diff --git a/tensorflow/tools/benchmark/BUILD b/tensorflow/tools/benchmark/BUILD index ef7aa2398e9..e926a456732 100644 --- a/tensorflow/tools/benchmark/BUILD +++ b/tensorflow/tools/benchmark/BUILD @@ -61,7 +61,7 @@ tf_cc_test( # --host_crosstool_top=@bazel_tools//tools/cpp:toolchain # # NOTE: currently '-pthread' must be removed from the LINK_OPTS variable -# in google/protobuf/BUILD to sucessfully build for Android. This is temporary +# in @protobuf//:BUILD to sucessfully build for Android. This is temporary # pending an update of the version of the protobuf library that Tensorflow # uses. cc_binary( diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index dbf89e5bc67..0253f1e2189 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -27,10 +27,10 @@ sh_binary( "MANIFEST.in", "README", "setup.py", - "//tensorflow/core:framework_headers", ":other_headers", ":simple_console", "//tensorflow:tensorflow_py", + "//tensorflow/core:framework_headers", "//tensorflow/examples/tutorials/mnist:package", "//tensorflow/models/embedding:package", "//tensorflow/models/image/alexnet:all_files", diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh index 7f123937e8b..fd60d1da6e6 100755 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -61,8 +61,9 @@ function main() { # protobuf pip package doesn't ship with header files. Copy the headers # over so user defined ops can be compiled. + mkdir -p ${TMPDIR}/google rsync --include "*/" --include "*.h" --exclude "*" --prune-empty-dirs -a \ - $RUNFILES/google ${TMPDIR} + $RUNFILES/external/protobuf ${TMPDIR}/google rsync -a $RUNFILES/third_party/eigen3 ${TMPDIR}/third_party cp tensorflow/tools/pip_package/MANIFEST.in ${TMPDIR} diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index b95f84ce5e4..5632f6850b7 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -4,13 +4,6 @@ # within the workspace (e.g. "tensorflow/"), and tf_repo_name is the name of the # local_repository rule (e.g. "@tf"). def tf_workspace(path_prefix = "", tf_repo_name = ""): - native.new_http_archive( - name = "gmock_archive", - url = "https://archive.openswitch.net/gmock-1.7.0.zip", - sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b", - build_file = path_prefix + "google/protobuf/gmock.BUILD", - ) - native.new_http_archive( name = "eigen_archive", url = "https://bitbucket.org/eigen/eigen/get/f3a13643ac1f.tar.gz", @@ -18,16 +11,6 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): build_file = path_prefix + "eigen.BUILD", ) - native.bind( - name = "gtest", - actual = "@gmock_archive//:gtest", - ) - - native.bind( - name = "gtest_main", - actual = "@gmock_archive//:gtest_main", - ) - native.git_repository( name = "re2", remote = "https://github.com/google/re2.git", @@ -85,16 +68,44 @@ def tf_workspace(path_prefix = "", tf_repo_name = ""): actual = "@six_archive//:six", ) + native.git_repository( + name = "protobuf", + remote = "https://github.com/google/protobuf", + commit = "ed87c1fe2c6e1633cadb62cf54b2723b2b25c280", + ) + + native.new_http_archive( + name = "gmock_archive", + url = "https://archive.openswitch.net/gmock-1.7.0.zip", + sha256 = "26fcbb5925b74ad5fc8c26b0495dfc96353f4d553492eb97e85a8a6d2f43095b", + build_file = path_prefix + "gmock.BUILD", + ) + + native.bind( + name = "gtest", + actual = "@gmock_archive//:gtest", + ) + + native.bind( + name = "gtest_main", + actual = "@gmock_archive//:gtest_main", + ) + + native.bind( + name = "python_headers", + actual = tf_repo_name + "//util/python:python_headers", + ) + # grpc expects //external:protobuf_clib and //external:protobuf_compiler # to point to the protobuf's compiler library. native.bind( name = "protobuf_clib", - actual = tf_repo_name + "//google/protobuf:protoc_lib", + actual = "@protobuf//:protoc_lib", ) native.bind( name = "protobuf_compiler", - actual = tf_repo_name + "//google/protobuf:protoc_lib", + actual = "@protobuf//:protoc_lib", ) native.git_repository(