Upgrade gRPC version used in OSS. Fix-forward of #14262. Remove namespace-compatibility hack.

PiperOrigin-RevId: 175586521

configure.py

@@ -976,6 +976,10 @@ def create_android_bazelrc_configs():
   write_to_bazelrc('build:android_arm64 --cpu=arm64-v8a')
 
 
+def set_grpc_build_flags():
+  write_to_bazelrc('build --define grpc_no_ares=true')
+
+
 def main():
   # Make a copy of os.environ to be clear when functions and getting and setting
   # environment variables.
@@ -1043,6 +1047,7 @@ def main():
     set_mpi_home(environ_cp)
     set_other_mpi_vars(environ_cp)
 
+  set_grpc_build_flags()
   set_cc_opt_flags(environ_cp)
   set_mkl()
   set_monolithic()

tensorflow/contrib/cmake/external/grpc.cmake

@@ -17,7 +17,7 @@ include (ExternalProject)
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
 set(GRPC_URL https://github.com/grpc/grpc.git)
 set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
-set(GRPC_TAG 781fd6f6ea03645a520cd5c675da67ab61f87e4b)
+set(GRPC_TAG 54e8f37e537794c2d814c1604c1282125f64f093)
 
 if(WIN32)
   set(grpc_STATIC_LIBRARIES
@@ -28,10 +28,11 @@ else()
   set(grpc_STATIC_LIBRARIES
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++_unsecure.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc_unsecure.a
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a
+      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a)
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/libcares.a)
 endif()
 
+add_definitions(-DGRPC_ARES=0)
+
 ExternalProject_Add(grpc
     PREFIX grpc
     DEPENDS protobuf zlib
@@ -39,9 +40,6 @@ ExternalProject_Add(grpc
     GIT_TAG ${GRPC_TAG}
     DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
     BUILD_IN_SOURCE 1
-    # TODO(jhseu): Remove this PATCH_COMMAND once grpc removes the dependency
-    # on "grpc" from the "grpc++_unsecure" rule.
-    PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/grpc/CMakeLists.txt ${GRPC_BUILD}
     BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release --target grpc++_unsecure
     COMMAND ${CMAKE_COMMAND} --build . --config Release --target grpc_cpp_plugin
     INSTALL_COMMAND ""

tensorflow/contrib/verbs/grpc_verbs_service_impl.cc

@@ -43,22 +43,21 @@ VerbsService::Stub::Stub(
     const std::shared_ptr< ::grpc::ChannelInterface>& channel)
     : channel_(channel),
       rpcmethod_GetRemoteAddress_(grpcVerbsService_method_names[0],
-                                  ::grpc::RpcMethod::NORMAL_RPC,
+                                  ::grpc::internal::RpcMethod::NORMAL_RPC,
                                   channel) {}
 
 ::grpc::Status VerbsService::Stub::GetRemoteAddress(
     ::grpc::ClientContext* context, const GetRemoteAddressRequest& request,
     GetRemoteAddressResponse* response) {
-  return ::grpc::BlockingUnaryCall(
+  return ::grpc::internal::BlockingUnaryCall(
       channel_.get(), rpcmethod_GetRemoteAddress_, context, request, response);
 }
 
 VerbsService::AsyncService::AsyncService() {
   for (int i = 0; i < 1; ++i) {
-    AddMethod(new ::grpc::RpcServiceMethod(
+    AddMethod(new ::grpc::internal::RpcServiceMethod(
         grpcVerbsService_method_names[i],
-        ::grpc::RpcMethod::NORMAL_RPC,
+        ::grpc::internal::RpcMethod::NORMAL_RPC, nullptr));
-        nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
 }

tensorflow/contrib/verbs/grpc_verbs_service_impl.h

@@ -28,15 +28,6 @@ limitations under the License.
 #include "tensorflow/contrib/verbs/verbs_service.pb.h"
 
 namespace grpc {
-
-// ensure internal namespace exists
-namespace internal {
-// bring in contents of external namespace
-using namespace ::grpc;
-}  // namespace internal
-// bring in contents of internal namespace
-using namespace internal;
-
 class CompletionQueue;
 class Channel;
 class RpcService;
@@ -70,7 +61,7 @@ class VerbsService GRPC_FINAL {
 
    private:
     std::shared_ptr< ::grpc::ChannelInterface> channel_;
-    const ::grpc::RpcMethod rpcmethod_GetRemoteAddress_;
+    const ::grpc::internal::RpcMethod rpcmethod_GetRemoteAddress_;
   };
   static std::unique_ptr<Stub> NewStub(
       const std::shared_ptr< ::grpc::ChannelInterface>& channel,

tensorflow/core/distributed_runtime/rpc/BUILD

@@ -200,7 +200,6 @@ cc_library(
     srcs = ["grpc_worker_service_impl.cc"],
     hdrs = ["grpc_worker_service_impl.h"],
     deps = [
-        ":grpc_namespace_compat",
         ":grpc_serialization_traits",
         "//tensorflow/core:worker_proto_cc",
         "//tensorflow/core/distributed_runtime:tensor_coding",
@@ -247,22 +246,12 @@ cc_library(
     srcs = ["grpc_master_service_impl.cc"],
     hdrs = ["grpc_master_service_impl.h"],
     deps = [
-        ":grpc_namespace_compat",
         ":grpc_serialization_traits",
         "//tensorflow/core:master_proto_cc",
         "@grpc//:grpc++_unsecure",
     ],
 )
 
-cc_library(
-    name = "grpc_namespace_compat",
-    srcs = [],
-    hdrs = ["grpc_namespace_compat.h"],
-    deps = [
-        "@grpc//:grpc++_unsecure",
-    ],
-)
-
 cc_library(
     name = "grpc_serialization_traits",
     srcs = [],

tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc

@@ -49,75 +49,77 @@ MasterService::Stub::Stub(
     const std::shared_ptr< ::grpc::ChannelInterface>& channel)
     : channel_(channel),
       rpcmethod_CreateSession_(grpcMasterService_method_names[0],
-                               ::grpc::RpcMethod::NORMAL_RPC, channel),
+                               ::grpc::internal::RpcMethod::NORMAL_RPC,
+                               channel),
       rpcmethod_ExtendSession_(grpcMasterService_method_names[1],
-                               ::grpc::RpcMethod::NORMAL_RPC, channel),
+                               ::grpc::internal::RpcMethod::NORMAL_RPC,
+                               channel),
       rpcmethod_PartialRunSetup_(grpcMasterService_method_names[2],
-                                 ::grpc::RpcMethod::NORMAL_RPC, channel),
+                                 ::grpc::internal::RpcMethod::NORMAL_RPC,
+                                 channel),
       rpcmethod_RunStep_(grpcMasterService_method_names[3],
-                         ::grpc::RpcMethod::NORMAL_RPC, channel),
+                         ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_CloseSession_(grpcMasterService_method_names[4],
-                              ::grpc::RpcMethod::NORMAL_RPC, channel),
+                              ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_ListDevices_(grpcMasterService_method_names[5],
-                             ::grpc::RpcMethod::NORMAL_RPC, channel),
+                             ::grpc::internal::RpcMethod::NORMAL_RPC, channel),
       rpcmethod_Reset_(grpcMasterService_method_names[6],
-                       ::grpc::RpcMethod::NORMAL_RPC, channel) {}
+                       ::grpc::internal::RpcMethod::NORMAL_RPC, channel) {}
 
 ::grpc::Status MasterService::Stub::CreateSession(
     ::grpc::ClientContext* context, const CreateSessionRequest& request,
     CreateSessionResponse* response) {
-  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_,
+  return ::grpc::internal::BlockingUnaryCall(
-                                   context, request, response);
+      channel_.get(), rpcmethod_CreateSession_, context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::ExtendSession(
     ::grpc::ClientContext* context, const ExtendSessionRequest& request,
     ExtendSessionResponse* response) {
-  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_,
+  return ::grpc::internal::BlockingUnaryCall(
-                                   context, request, response);
+      channel_.get(), rpcmethod_ExtendSession_, context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::PartialRunSetup(
     ::grpc::ClientContext* context, const PartialRunSetupRequest& request,
     PartialRunSetupResponse* response) {
-  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_,
+  return ::grpc::internal::BlockingUnaryCall(
-                                   context, request, response);
+      channel_.get(), rpcmethod_PartialRunSetup_, context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::RunStep(::grpc::ClientContext* context,
                                             const RunStepRequest& request,
                                             RunStepResponse* response) {
-  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context,
+  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_,
-                                   request, response);
+                                             context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::CloseSession(
     ::grpc::ClientContext* context, const CloseSessionRequest& request,
     CloseSessionResponse* response) {
-  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_,
+  return ::grpc::internal::BlockingUnaryCall(
-                                   context, request, response);
+      channel_.get(), rpcmethod_CloseSession_, context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::ListDevices(
     ::grpc::ClientContext* context, const ListDevicesRequest& request,
     ListDevicesResponse* response) {
-  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_,
+  return ::grpc::internal::BlockingUnaryCall(
-                                   context, request, response);
+      channel_.get(), rpcmethod_ListDevices_, context, request, response);
 }
 
 ::grpc::Status MasterService::Stub::Reset(::grpc::ClientContext* context,
                                           const ResetRequest& request,
                                           ResetResponse* response) {
-  return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context,
+  return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_,
-                                   request, response);
+                                             context, request, response);
 }
 
 MasterService::AsyncService::AsyncService() {
   for (int i = 0; i < 7; ++i) {
-    AddMethod(new ::grpc::RpcServiceMethod(
+    AddMethod(new ::grpc::internal::RpcServiceMethod(
         grpcMasterService_method_names[i],
-        ::grpc::RpcMethod::NORMAL_RPC,
+        ::grpc::internal::RpcMethod::NORMAL_RPC, nullptr));
-        nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
 }

tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h

@@ -25,7 +25,6 @@ limitations under the License.
 #include "grpc++/impl/codegen/stub_options.h"
 #include "grpc++/impl/codegen/sync_stream.h"
 
-#include "tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h"
 #include "tensorflow/core/protobuf/master.pb.h"
 
@@ -108,13 +107,13 @@ class MasterService final {
 
    private:
     std::shared_ptr< ::grpc::ChannelInterface> channel_;
-    const ::grpc::RpcMethod rpcmethod_CreateSession_;
+    const ::grpc::internal::RpcMethod rpcmethod_CreateSession_;
-    const ::grpc::RpcMethod rpcmethod_ExtendSession_;
+    const ::grpc::internal::RpcMethod rpcmethod_ExtendSession_;
-    const ::grpc::RpcMethod rpcmethod_PartialRunSetup_;
+    const ::grpc::internal::RpcMethod rpcmethod_PartialRunSetup_;
-    const ::grpc::RpcMethod rpcmethod_RunStep_;
+    const ::grpc::internal::RpcMethod rpcmethod_RunStep_;
-    const ::grpc::RpcMethod rpcmethod_CloseSession_;
+    const ::grpc::internal::RpcMethod rpcmethod_CloseSession_;
-    const ::grpc::RpcMethod rpcmethod_ListDevices_;
+    const ::grpc::internal::RpcMethod rpcmethod_ListDevices_;
-    const ::grpc::RpcMethod rpcmethod_Reset_;
+    const ::grpc::internal::RpcMethod rpcmethod_Reset_;
   };
   static std::unique_ptr<Stub> NewStub(
       const std::shared_ptr< ::grpc::ChannelInterface>& channel,

tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h

@@ -1,32 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_
-#define THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_
-
-// This file is a transitional place-holder until gRPC versions consistently
-// use namespace grpc::internal for library-internal structures
-
-namespace grpc {
-// ensure internal namespace exists
-namespace internal {
-// bring in contents of external namespace
-using namespace ::grpc;
-}  // namespace internal
-// bring in contents of internal namespace
-using namespace internal;
-}  // namespace grpc
-
-#endif  // THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_

tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc

@@ -58,9 +58,9 @@ namespace grpc {
 
 WorkerService::AsyncService::AsyncService() {
   for (int i = 0; i < kGrpcNumWorkerMethods; ++i) {
-    AddMethod(new ::grpc::RpcServiceMethod(
+    AddMethod(new ::grpc::internal::RpcServiceMethod(
         GrpcWorkerMethodName(static_cast<GrpcWorkerMethod>(i)),
-        ::grpc::RpcMethod::NORMAL_RPC, nullptr));
+        ::grpc::internal::RpcMethod::NORMAL_RPC, nullptr));
     ::grpc::Service::MarkMethodAsync(i);
   }
 }

tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h

@@ -26,7 +26,6 @@ limitations under the License.
 #include "grpc++/impl/codegen/sync_stream.h"
 #include "grpc++/support/byte_buffer.h"
 
-#include "tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h"
 #include "tensorflow/core/distributed_runtime/tensor_coding.h"
 #include "tensorflow/core/protobuf/worker.pb.h"

tensorflow/workspace.bzl

@@ -416,6 +416,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       actual = "@protobuf_archive//:protobuf",
   )
 
+  native.bind(
+      name = "protobuf_headers",
+      actual = "@protobuf_archive//:protobuf_headers",
+  )
+
   # We need to import the protobuf library under the names com_google_protobuf
   # and com_google_protobuf_cc to enable proto_library support in bazel.
   # Unfortunately there is no way to alias http_archives at the moment.
@@ -528,15 +533,21 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       actual = "@grpc//third_party/nanopb:nanopb",
   )
 
-  patched_http_archive(
+  native.http_archive(
       name = "grpc",
       urls = [
-          "https://mirror.bazel.build/github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz",
+          # "https://mirror.bazel.build/github.com/grpc/grpc/archive/54e8f37e537794c2d814c1604c1282125f64f093.tar.gz",
-          # "https://github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz",
+          "https://github.com/grpc/grpc/archive/54e8f37e537794c2d814c1604c1282125f64f093.tar.gz",
       ],
-      sha256 = "2004635e6a078acfac8ffa71738397796be4f8fb72f572cc44ecee5d99511d9f",
+     sha256 = "c2166b6d96daddf72fe45b2c594210c65ca17ec3c1b2e12089159a9529edb5e4",
-      strip_prefix = "grpc-781fd6f6ea03645a520cd5c675da67ab61f87e4b",
+     strip_prefix = "grpc-54e8f37e537794c2d814c1604c1282125f64f093",
-      patch_file = str(Label("//third_party/grpc:grpc.patch")),
+  )
+
+  # gRPC wants the existence of a cares dependence but its contents are not
+  # actually important since we have set GRPC_ARES=0 in tools/bazel.rc
+  native.bind(
+      name = "cares",
+      actual = "@grpc//third_party/nanopb:nanopb",
   )
 
   # protobuf expects //external:grpc_cpp_plugin to point to grpc's

third_party/grpc/grpc.patch

@@ -1,105 +0,0 @@
-diff --git a/BUILD b/BUILD
-index 6552d5879e..59adb1ce1c 100644
---- a/BUILD
-+++ b/BUILD
-@@ -287,6 +287,7 @@ grpc_cc_library(
-         "grpc++_base_unsecure",
-         "grpc++_codegen_base",
-         "grpc++_codegen_base_src",
-+        "grpc++_codegen_proto",
-         "grpc_unsecure",
-     ],
- )
-@@ -1519,13 +1520,13 @@ grpc_cc_library(
- 
- grpc_cc_library(
-     name = "grpc++_config_proto",
--    external_deps = [
--        "protobuf",
--    ],
-     language = "c++",
-     public_hdrs = [
-         "include/grpc++/impl/codegen/config_protobuf.h",
-     ],
-+    deps = [
-+        "@protobuf_archive//:protobuf_headers",
-+    ],
- )
- 
- grpc_cc_library(
-diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
-index f793cae56d..0295adb8ab 100644
---- a/bazel/grpc_build_system.bzl
-+++ b/bazel/grpc_build_system.bzl
-@@ -80,7 +80,7 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
-     linkopts = ["-pthread"],
-   )
- 
--def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False):
-+def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False, linkopts = []):
-   copts = []
-   if language.upper() == "C":
-     copts = ["-std=c99"]
-@@ -93,7 +93,7 @@ def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], da
-     linkshared = linkshared,
-     deps = deps + ["//external:" + dep for dep in external_deps],
-     copts = copts,
--    linkopts = ["-pthread"],
-+    linkopts = ["-pthread"] + linkopts,
-   )
- 
- def grpc_generate_one_off_targets():
-diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
-index 7eb599d81a..4cc2e30af4 100644
---- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
-+++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
-@@ -28,18 +28,12 @@ extern void grpc_client_channel_init(void);
- extern void grpc_client_channel_shutdown(void);
- extern void grpc_inproc_plugin_init(void);
- extern void grpc_inproc_plugin_shutdown(void);
--extern void grpc_resolver_dns_ares_init(void);
--extern void grpc_resolver_dns_ares_shutdown(void);
- extern void grpc_resolver_dns_native_init(void);
- extern void grpc_resolver_dns_native_shutdown(void);
- extern void grpc_resolver_sockaddr_init(void);
- extern void grpc_resolver_sockaddr_shutdown(void);
--extern void grpc_resolver_fake_init(void);
--extern void grpc_resolver_fake_shutdown(void);
- extern void grpc_load_reporting_plugin_init(void);
- extern void grpc_load_reporting_plugin_shutdown(void);
--extern void grpc_lb_policy_grpclb_init(void);
--extern void grpc_lb_policy_grpclb_shutdown(void);
- extern void grpc_lb_policy_pick_first_init(void);
- extern void grpc_lb_policy_pick_first_shutdown(void);
- extern void grpc_lb_policy_round_robin_init(void);
-@@ -64,18 +58,12 @@ void grpc_register_built_in_plugins(void) {
-                        grpc_client_channel_shutdown);
-   grpc_register_plugin(grpc_inproc_plugin_init,
-                        grpc_inproc_plugin_shutdown);
--  grpc_register_plugin(grpc_resolver_dns_ares_init,
--                       grpc_resolver_dns_ares_shutdown);
-   grpc_register_plugin(grpc_resolver_dns_native_init,
-                        grpc_resolver_dns_native_shutdown);
-   grpc_register_plugin(grpc_resolver_sockaddr_init,
-                        grpc_resolver_sockaddr_shutdown);
--  grpc_register_plugin(grpc_resolver_fake_init,
--                       grpc_resolver_fake_shutdown);
-   grpc_register_plugin(grpc_load_reporting_plugin_init,
-                        grpc_load_reporting_plugin_shutdown);
--  grpc_register_plugin(grpc_lb_policy_grpclb_init,
--                       grpc_lb_policy_grpclb_shutdown);
-   grpc_register_plugin(grpc_lb_policy_pick_first_init,
-                        grpc_lb_policy_pick_first_shutdown);
-   grpc_register_plugin(grpc_lb_policy_round_robin_init,
-diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD
-index 33240f6f69..d2e1f67f06 100644
---- a/test/cpp/util/BUILD
-+++ b/test/cpp/util/BUILD
-@@ -29,6 +29,7 @@ package(
- grpc_cc_binary(
-     name = "testso.so",
-     srcs = [],
-+    linkopts = ['-Wl,--no-undefined'],
-     linkshared = 1,
-     deps = ["//:grpc++_unsecure"],
- )

tools/bazel.rc

@@ -19,6 +19,7 @@ build:sycl_asan --define=using_sycl=true --copt -fno-omit-frame-pointer --copt -
 
 build --define=use_fast_cpp_protos=true
 build --define=allow_oversize_protos=true
+build --define=grpc_no_ares=true
 
 build --spawn_strategy=standalone
 build --genrule_strategy=standalone