Upgrade gRPC (#11768)

* BUILD rule modifications

* More build fixes

* Code changes

* More code fixes

* Working tests

* CMake build

* Fix pprof

* Fix header includes

* CMake fix test

* Bazel clean

* Fix verbs

* More verbs fixes

* bazel clean for XLA

* Windows build fix test

* Add openssl/rand.h

* New cmake build command

* --config Release
This commit is contained in:
Jonathan Hseu 2017-07-27 10:40:16 -07:00 committed by Vijay Vasudevan
parent 3cd828474f
commit d09304fca4
40 changed files with 14465 additions and 2768 deletions

View File

@ -18,7 +18,7 @@ cc_library(
"//tensorflow/compiler/xla/service",
"//third_party/eigen3",
"@local_config_cuda//cuda:cuda_headers",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
)

View File

@ -1,15 +1,15 @@
"""Wrapper around cc_proto_library used inside the XLA codebase."""
load("@protobuf//:protobuf.bzl", "cc_proto_library")
load("@protobuf_archive//:protobuf.bzl", "cc_proto_library")
# xla_proto_library() is a convenience wrapper around cc_proto_library.
def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0):
cc_proto_library(name=name,
srcs=srcs,
deps=deps,
cc_libs = ["@protobuf//:protobuf"],
protoc="@protobuf//:protoc",
default_runtime="@protobuf//:protobuf",
cc_libs = ["@protobuf_archive//:protobuf"],
protoc="@protobuf_archive//:protoc",
default_runtime="@protobuf_archive//:protobuf",
testonly=testonly,
visibility=visibility,)

View File

@ -17,7 +17,7 @@ include (ExternalProject)
set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
set(GRPC_URL https://github.com/grpc/grpc.git)
set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
set(GRPC_TAG 3bc78cd0b5bd784a235c01612d634b1ec5f8fb97)
set(GRPC_TAG 781fd6f6ea03645a520cd5c675da67ab61f87e4b)
if(WIN32)
set(grpc_STATIC_LIBRARIES
@ -38,7 +38,10 @@ ExternalProject_Add(grpc
GIT_TAG ${GRPC_TAG}
DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
BUILD_IN_SOURCE 1
# TODO(jhseu): Remove this PATCH_COMMAND once grpc removes the dependency
# on "grpc" from the "grpc++_unsecure" rule.
PATCH_COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/grpc/CMakeLists.txt ${GRPC_BUILD}
BUILD_COMMAND ${CMAKE_COMMAND} --build . --config Release --target grpc++_unsecure
INSTALL_COMMAND ""
CMAKE_CACHE_ARGS
-DCMAKE_BUILD_TYPE:STRING=Release
@ -46,5 +49,13 @@ ExternalProject_Add(grpc
-DPROTOBUF_INCLUDE_DIRS:STRING=${PROTOBUF_INCLUDE_DIRS}
-DPROTOBUF_LIBRARIES:STRING=${protobuf_STATIC_LIBRARIES}
-DZLIB_ROOT:STRING=${ZLIB_INSTALL}
-DgRPC_SSL_PROVIDER:STRING=NONE
)
# grpc/src/core/ext/census/tracing.c depends on the existence of openssl/rand.h.
ExternalProject_Add_Step(grpc copy_rand
COMMAND ${CMAKE_COMMAND} -E copy
${CMAKE_SOURCE_DIR}/patches/grpc/rand.h ${GRPC_BUILD}/include/openssl/rand.h
DEPENDEES patch
DEPENDERS build
)

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,14 @@
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

View File

@ -12,7 +12,7 @@ cc_library(
":clustering_ops",
":masked_matmul_ops",
":wals_solver_ops",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
)
@ -22,7 +22,7 @@ cc_library(
deps = [
"//tensorflow/core:framework_headers_lib",
"//third_party/eigen3",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
alwayslink = 1,
)
@ -33,7 +33,7 @@ cc_library(
deps = [
"//tensorflow/core:framework_headers_lib",
"//third_party/eigen3",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
alwayslink = 1,
)
@ -45,7 +45,7 @@ cc_library(
"//tensorflow/core:framework_headers_lib",
"//tensorflow/core/kernels:bounds_check",
"//third_party/eigen3",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
alwayslink = 1,
)

View File

@ -17,7 +17,7 @@ cc_library(
],
deps = [
"//tensorflow/core:framework_headers_lib",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
)

View File

@ -106,7 +106,7 @@ tf_custom_op_library(
"ops/single_image_random_dot_stereograms_ops.cc",
],
deps = [
"@protobuf//:protobuf",
"@protobuf_archive//:protobuf",
],
)

View File

@ -13,7 +13,7 @@ cc_library(
deps = [
"//tensorflow/core:framework_headers_lib",
"//third_party/eigen3",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
alwayslink = 1,
)

View File

@ -14,7 +14,7 @@ cc_library(
"//tensorflow/core:framework_headers_lib",
"//third_party/eigen3",
"@farmhash_archive//:farmhash",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
alwayslink = 1,
)

View File

@ -53,7 +53,7 @@ cc_library(
":tree_utils",
"//tensorflow/core:framework_headers_lib",
"//third_party/eigen3",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
alwayslink = 1,
)
@ -356,7 +356,7 @@ cc_library(
deps = [
"//tensorflow/core:framework_headers_lib",
"//third_party/eigen3",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
)

View File

@ -88,7 +88,7 @@ cc_library(
deps = [
"//tensorflow/core:framework_headers_lib",
"//third_party/eigen3",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
)

View File

@ -43,20 +43,22 @@ VerbsService::Stub::Stub(
const std::shared_ptr< ::grpc::ChannelInterface>& channel)
: channel_(channel),
rpcmethod_GetRemoteAddress_(grpcVerbsService_method_names[0],
::grpc::RpcMethod::NORMAL_RPC, channel) {}
::grpc::internal::RpcMethod::NORMAL_RPC,
channel) {}
::grpc::Status VerbsService::Stub::GetRemoteAddress(
::grpc::ClientContext* context, const GetRemoteAddressRequest& request,
GetRemoteAddressResponse* response) {
return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_GetRemoteAddress_,
context, request, response);
return ::grpc::internal::BlockingUnaryCall(
channel_.get(), rpcmethod_GetRemoteAddress_, context, request, response);
}
VerbsService::AsyncService::AsyncService() {
for (int i = 0; i < 1; ++i) {
AddMethod(new ::grpc::RpcServiceMethod(grpcVerbsService_method_names[i],
::grpc::RpcMethod::NORMAL_RPC,
nullptr));
AddMethod(new ::grpc::internal::RpcServiceMethod(
grpcVerbsService_method_names[i],
::grpc::internal::RpcMethod::NORMAL_RPC,
nullptr));
::grpc::Service::MarkMethodAsync(i);
}
}

View File

@ -61,7 +61,7 @@ class VerbsService GRPC_FINAL {
private:
std::shared_ptr< ::grpc::ChannelInterface> channel_;
const ::grpc::RpcMethod rpcmethod_GetRemoteAddress_;
const ::grpc::internal::RpcMethod rpcmethod_GetRemoteAddress_;
};
static std::unique_ptr<Stub> NewStub(
const std::shared_ptr< ::grpc::ChannelInterface>& channel,

View File

@ -49,74 +49,75 @@ MasterService::Stub::Stub(
const std::shared_ptr< ::grpc::ChannelInterface>& channel)
: channel_(channel),
rpcmethod_CreateSession_(grpcMasterService_method_names[0],
::grpc::RpcMethod::NORMAL_RPC, channel),
::grpc::internal::RpcMethod::NORMAL_RPC, channel),
rpcmethod_ExtendSession_(grpcMasterService_method_names[1],
::grpc::RpcMethod::NORMAL_RPC, channel),
::grpc::internal::RpcMethod::NORMAL_RPC, channel),
rpcmethod_PartialRunSetup_(grpcMasterService_method_names[2],
::grpc::RpcMethod::NORMAL_RPC, channel),
::grpc::internal::RpcMethod::NORMAL_RPC, channel),
rpcmethod_RunStep_(grpcMasterService_method_names[3],
::grpc::RpcMethod::NORMAL_RPC, channel),
::grpc::internal::RpcMethod::NORMAL_RPC, channel),
rpcmethod_CloseSession_(grpcMasterService_method_names[4],
::grpc::RpcMethod::NORMAL_RPC, channel),
::grpc::internal::RpcMethod::NORMAL_RPC, channel),
rpcmethod_ListDevices_(grpcMasterService_method_names[5],
::grpc::RpcMethod::NORMAL_RPC, channel),
::grpc::internal::RpcMethod::NORMAL_RPC, channel),
rpcmethod_Reset_(grpcMasterService_method_names[6],
::grpc::RpcMethod::NORMAL_RPC, channel) {}
::grpc::internal::RpcMethod::NORMAL_RPC, channel) {}
::grpc::Status MasterService::Stub::CreateSession(
::grpc::ClientContext* context, const CreateSessionRequest& request,
CreateSessionResponse* response) {
return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_,
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_,
context, request, response);
}
::grpc::Status MasterService::Stub::ExtendSession(
::grpc::ClientContext* context, const ExtendSessionRequest& request,
ExtendSessionResponse* response) {
return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_,
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_,
context, request, response);
}
::grpc::Status MasterService::Stub::PartialRunSetup(
::grpc::ClientContext* context, const PartialRunSetupRequest& request,
PartialRunSetupResponse* response) {
return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_,
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_,
context, request, response);
}
::grpc::Status MasterService::Stub::RunStep(::grpc::ClientContext* context,
const RunStepRequest& request,
RunStepResponse* response) {
return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context,
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context,
request, response);
}
::grpc::Status MasterService::Stub::CloseSession(
::grpc::ClientContext* context, const CloseSessionRequest& request,
CloseSessionResponse* response) {
return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_,
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_,
context, request, response);
}
::grpc::Status MasterService::Stub::ListDevices(
::grpc::ClientContext* context, const ListDevicesRequest& request,
ListDevicesResponse* response) {
return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_,
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_,
context, request, response);
}
::grpc::Status MasterService::Stub::Reset(::grpc::ClientContext* context,
const ResetRequest& request,
ResetResponse* response) {
return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context,
return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context,
request, response);
}
MasterService::AsyncService::AsyncService() {
for (int i = 0; i < 7; ++i) {
AddMethod(new ::grpc::RpcServiceMethod(grpcMasterService_method_names[i],
::grpc::RpcMethod::NORMAL_RPC,
nullptr));
AddMethod(new ::grpc::internal::RpcServiceMethod(
grpcMasterService_method_names[i],
::grpc::internal::RpcMethod::NORMAL_RPC,
nullptr));
::grpc::Service::MarkMethodAsync(i);
}
}

View File

@ -53,7 +53,7 @@ namespace grpc {
// definition in "//tensorflow/core/protobuf/master_service.proto",
// and the gRPC generated stub and service classes.
// See that file for the definition of methods and messages.
class MasterService GRPC_FINAL {
class MasterService final {
public:
class StubInterface {
public:
@ -80,40 +80,40 @@ class MasterService GRPC_FINAL {
const ResetRequest& request,
ResetResponse* response) = 0;
};
class Stub GRPC_FINAL : public StubInterface {
class Stub final : public StubInterface {
public:
Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel);
::grpc::Status CreateSession(::grpc::ClientContext* context,
const CreateSessionRequest& request,
CreateSessionResponse* response) GRPC_OVERRIDE;
CreateSessionResponse* response) override;
::grpc::Status ExtendSession(::grpc::ClientContext* context,
const ExtendSessionRequest& request,
ExtendSessionResponse* response) GRPC_OVERRIDE;
ExtendSessionResponse* response) override;
::grpc::Status PartialRunSetup(
::grpc::ClientContext* context, const PartialRunSetupRequest& request,
PartialRunSetupResponse* response) GRPC_OVERRIDE;
PartialRunSetupResponse* response) override;
::grpc::Status RunStep(::grpc::ClientContext* context,
const RunStepRequest& request,
RunStepResponse* response) GRPC_OVERRIDE;
RunStepResponse* response) override;
::grpc::Status CloseSession(::grpc::ClientContext* context,
const CloseSessionRequest& request,
CloseSessionResponse* response) GRPC_OVERRIDE;
CloseSessionResponse* response) override;
::grpc::Status ListDevices(::grpc::ClientContext* context,
const ListDevicesRequest& request,
ListDevicesResponse* response) GRPC_OVERRIDE;
ListDevicesResponse* response) override;
::grpc::Status Reset(::grpc::ClientContext* context,
const ResetRequest& request,
ResetResponse* response) GRPC_OVERRIDE;
ResetResponse* response) override;
private:
std::shared_ptr< ::grpc::ChannelInterface> channel_;
const ::grpc::RpcMethod rpcmethod_CreateSession_;
const ::grpc::RpcMethod rpcmethod_ExtendSession_;
const ::grpc::RpcMethod rpcmethod_PartialRunSetup_;
const ::grpc::RpcMethod rpcmethod_RunStep_;
const ::grpc::RpcMethod rpcmethod_CloseSession_;
const ::grpc::RpcMethod rpcmethod_ListDevices_;
const ::grpc::RpcMethod rpcmethod_Reset_;
const ::grpc::internal::RpcMethod rpcmethod_CreateSession_;
const ::grpc::internal::RpcMethod rpcmethod_ExtendSession_;
const ::grpc::internal::RpcMethod rpcmethod_PartialRunSetup_;
const ::grpc::internal::RpcMethod rpcmethod_RunStep_;
const ::grpc::internal::RpcMethod rpcmethod_CloseSession_;
const ::grpc::internal::RpcMethod rpcmethod_ListDevices_;
const ::grpc::internal::RpcMethod rpcmethod_Reset_;
};
static std::unique_ptr<Stub> NewStub(
const std::shared_ptr< ::grpc::ChannelInterface>& channel,

View File

@ -17,6 +17,7 @@ limitations under the License.
#define THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_SERIALIZATION_TRAITS_H_
#include "grpc++/impl/codegen/proto_utils.h"
#include "grpc++/support/slice.h"
namespace grpc {
@ -24,7 +25,7 @@ namespace tensorflow_helper {
const int kGrpcBufferWriterMaxBufferLength = 8192;
class GrpcBufferWriter GRPC_FINAL
class GrpcBufferWriter final
: public ::grpc::protobuf::io::ZeroCopyOutputStream {
public:
explicit GrpcBufferWriter(grpc_byte_buffer** bp, int block_size)
@ -33,35 +34,35 @@ class GrpcBufferWriter GRPC_FINAL
slice_buffer_ = &(*bp)->data.raw.slice_buffer;
}
~GrpcBufferWriter() GRPC_OVERRIDE {
~GrpcBufferWriter() override {
if (have_backup_) {
g_core_codegen_interface->gpr_slice_unref(backup_slice_);
g_core_codegen_interface->grpc_slice_unref(backup_slice_);
}
}
bool Next(void** data, int* size) GRPC_OVERRIDE {
bool Next(void** data, int* size) override {
if (have_backup_) {
slice_ = backup_slice_;
have_backup_ = false;
} else {
slice_ = g_core_codegen_interface->gpr_slice_malloc(block_size_);
slice_ = g_core_codegen_interface->grpc_slice_malloc(block_size_);
}
*data = GPR_SLICE_START_PTR(slice_);
*data = GRPC_SLICE_START_PTR(slice_);
// On win x64, int is only 32bit
GPR_CODEGEN_ASSERT(GPR_SLICE_LENGTH(slice_) <= INT_MAX);
byte_count_ += * size = (int)GPR_SLICE_LENGTH(slice_);
g_core_codegen_interface->gpr_slice_buffer_add(slice_buffer_, slice_);
GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX);
byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_);
g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_);
return true;
}
void BackUp(int count) GRPC_OVERRIDE {
g_core_codegen_interface->gpr_slice_buffer_pop(slice_buffer_);
void BackUp(int count) override {
g_core_codegen_interface->grpc_slice_buffer_pop(slice_buffer_);
if (count == block_size_) {
backup_slice_ = slice_;
} else {
backup_slice_ = g_core_codegen_interface->gpr_slice_split_tail(
&slice_, GPR_SLICE_LENGTH(slice_) - count);
g_core_codegen_interface->gpr_slice_buffer_add(slice_buffer_, slice_);
backup_slice_ = g_core_codegen_interface->grpc_slice_split_tail(
&slice_, GRPC_SLICE_LENGTH(slice_) - count);
g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_);
}
// It's dangerous to keep an inlined grpc_slice as the backup slice, since
// on a following Next() call, a reference will be returned to this slice
@ -71,18 +72,18 @@ class GrpcBufferWriter GRPC_FINAL
byte_count_ -= count;
}
grpc::protobuf::int64 ByteCount() const GRPC_OVERRIDE { return byte_count_; }
grpc::protobuf::int64 ByteCount() const override { return byte_count_; }
private:
const int block_size_;
int64_t byte_count_;
gpr_slice_buffer* slice_buffer_;
grpc_slice_buffer* slice_buffer_;
bool have_backup_;
gpr_slice backup_slice_;
gpr_slice slice_;
grpc_slice backup_slice_;
grpc_slice slice_;
};
class GrpcBufferReader GRPC_FINAL
class GrpcBufferReader final
: public ::grpc::protobuf::io::ZeroCopyInputStream {
typedef void (CoreCodegenInterface::*OldReaderInitAPI)(
grpc_byte_buffer_reader* reader, grpc_byte_buffer* buffer);
@ -104,13 +105,13 @@ class GrpcBufferReader GRPC_FINAL
ReaderInit(&CoreCodegenInterface::grpc_byte_buffer_reader_init, &reader_,
buffer);
}
~GrpcBufferReader() GRPC_OVERRIDE {
~GrpcBufferReader() override {
g_core_codegen_interface->grpc_byte_buffer_reader_destroy(&reader_);
}
bool Next(const void** data, int* size) GRPC_OVERRIDE {
bool Next(const void** data, int* size) override {
if (backup_count_ > 0) {
*data = GPR_SLICE_START_PTR(slice_) + GPR_SLICE_LENGTH(slice_) -
*data = GRPC_SLICE_START_PTR(slice_) + GRPC_SLICE_LENGTH(slice_) -
backup_count_;
GPR_CODEGEN_ASSERT(backup_count_ <= INT_MAX);
*size = (int)backup_count_;
@ -121,17 +122,17 @@ class GrpcBufferReader GRPC_FINAL
&slice_)) {
return false;
}
g_core_codegen_interface->gpr_slice_unref(slice_);
*data = GPR_SLICE_START_PTR(slice_);
g_core_codegen_interface->grpc_slice_unref(slice_);
*data = GRPC_SLICE_START_PTR(slice_);
// On win x64, int is only 32bit
GPR_CODEGEN_ASSERT(GPR_SLICE_LENGTH(slice_) <= INT_MAX);
byte_count_ += * size = (int)GPR_SLICE_LENGTH(slice_);
GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX);
byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_);
return true;
}
void BackUp(int count) GRPC_OVERRIDE { backup_count_ = count; }
void BackUp(int count) override { backup_count_ = count; }
bool Skip(int count) GRPC_OVERRIDE {
bool Skip(int count) override {
const void* data;
int size;
while (Next(&data, &size)) {
@ -146,7 +147,7 @@ class GrpcBufferReader GRPC_FINAL
return false;
}
grpc::protobuf::int64 ByteCount() const GRPC_OVERRIDE {
grpc::protobuf::int64 ByteCount() const override {
return byte_count_ - backup_count_;
}
@ -154,7 +155,7 @@ class GrpcBufferReader GRPC_FINAL
int64_t byte_count_;
int64_t backup_count_;
grpc_byte_buffer_reader reader_;
gpr_slice slice_;
grpc_slice slice_;
};
} // namespace tensorflow_helper
@ -175,12 +176,12 @@ class UnlimitedSizeProtoSerializationTraits {
return Status(StatusCode::INTERNAL, "Message length was negative");
} else if (byte_size <=
tensorflow_helper::kGrpcBufferWriterMaxBufferLength) {
gpr_slice slice = g_core_codegen_interface->gpr_slice_malloc(byte_size);
grpc_slice slice = g_core_codegen_interface->grpc_slice_malloc(byte_size);
GPR_CODEGEN_ASSERT(
GPR_SLICE_END_PTR(slice) ==
msg.SerializeWithCachedSizesToArray(GPR_SLICE_START_PTR(slice)));
GRPC_SLICE_END_PTR(slice) ==
msg.SerializeWithCachedSizesToArray(GRPC_SLICE_START_PTR(slice)));
*bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(&slice, 1);
g_core_codegen_interface->gpr_slice_unref(slice);
g_core_codegen_interface->grpc_slice_unref(slice);
return g_core_codegen_interface->ok();
} else {
tensorflow_helper::GrpcBufferWriter writer(

View File

@ -38,9 +38,9 @@ static void unref_tensorbuffer(void* raw) {
void EncodeRecvTensorResponseToByteBuffer(const RecvTensorResponse& proto,
::grpc::ByteBuffer* result) {
size_t len = proto.ByteSizeLong();
gpr_slice s = gpr_slice_malloc(len);
grpc_slice s = grpc_slice_malloc(len);
proto.SerializeWithCachedSizesToArray(
reinterpret_cast<uint8*>(GPR_SLICE_START_PTR(s)));
reinterpret_cast<uint8*>(GRPC_SLICE_START_PTR(s)));
::grpc::Slice slice(s, ::grpc::Slice::STEAL_REF);
*result = ::grpc::ByteBuffer(&slice, 1);
}
@ -68,12 +68,12 @@ void EncodeRecvTensorResponseToByteBuffer(const RecvTensorResponse& proto,
// E: <actual data for val's representation>
//
// If the tensor data is up to "kLargeTensorBytes", then A
// through E will all be encoded into "*result" in a single gpr_slice.
// through E will all be encoded into "*result" in a single grpc_slice.
//
// If the tensor data is larger than "kLargeTensorBytes", then A through
// D2 will be encoded in one gpr_slice, and E will be encoded in a second
// gpr_slice that points to the backing store for the tensor data, to avoid
// copying the tensor data (and the gpr_slice setup will be arrange so as
// D2 will be encoded in one grpc_slice, and E will be encoded in a second
// grpc_slice that points to the backing store for the tensor data, to avoid
// copying the tensor data (and the grpc_slice setup will be arrange so as
// to dereference the underlying tensor data buffer when it is no longer
// needed in the "*result" ByteBuffer).
static int VarLengthEncodingSize(uint32 tag, size_t bytes) {
@ -209,11 +209,11 @@ void EncodeTensorToByteBuffer(bool is_dead, const Tensor& val,
int num_slices = 0;
{
size_t slice_len = e.size() + (tensor_data_is_large ? 0 : tdata.size());
gpr_slice s0 = gpr_slice_malloc(slice_len);
memcpy(GPR_SLICE_START_PTR(s0), e.data(), e.size());
grpc_slice s0 = grpc_slice_malloc(slice_len);
memcpy(GRPC_SLICE_START_PTR(s0), e.data(), e.size());
if (!tensor_data_is_large) {
// (E)
memcpy(GPR_SLICE_START_PTR(s0) + e.size(), tdata.data(), tdata.size());
memcpy(GRPC_SLICE_START_PTR(s0) + e.size(), tdata.data(), tdata.size());
}
slices[0] = ::grpc::Slice(s0, ::grpc::Slice::STEAL_REF);
num_slices += 1;
@ -230,7 +230,7 @@ void EncodeTensorToByteBuffer(bool is_dead, const Tensor& val,
// hypothetical grpc_slice-related changes (e.g. the
// implementation could decide to destroy 0-length slices
// eagerly). In practice, this does not happen with the current
// implementation, and the gpr_slice interface at the moment does
// implementation, and the grpc_slice interface at the moment does
// not allow us to do the Tensor-unreferencing in the right way
// (since the Tensor pointer is different than the backing store
// array pointer).
@ -245,13 +245,13 @@ void EncodeTensorToByteBuffer(bool is_dead, const Tensor& val,
const TensorBuffer* buf = DMAHelper::buffer(&val);
buf->Ref();
gpr_slice s1 = gpr_slice_new(
grpc_slice s1 = grpc_slice_new(
const_cast<void*>(static_cast<const void*>(tdata.data())),
tdata.size(), do_nothing);
slices[1] = ::grpc::Slice(s1, ::grpc::Slice::STEAL_REF);
gpr_slice s2 =
gpr_slice_new(const_cast<TensorBuffer*>(buf), 0, unref_tensorbuffer);
grpc_slice s2 =
grpc_slice_new(const_cast<TensorBuffer*>(buf), 0, unref_tensorbuffer);
slices[2] = ::grpc::Slice(s2, ::grpc::Slice::STEAL_REF);
num_slices += 2;
}

View File

@ -80,9 +80,9 @@ grpc::protobuf::int64 GrpcByteBufferSource::ByteCount() const {
void GrpcUnparseProto(const protobuf::Message& src, grpc::ByteBuffer* dst) {
// TODO(sanjay): For bigger protos, serialize into a ZeroCopyOutputStream.
size_t len = src.ByteSizeLong();
gpr_slice s = gpr_slice_malloc(len);
grpc_slice s = grpc_slice_malloc(len);
src.SerializeWithCachedSizesToArray(
reinterpret_cast<uint8*>(GPR_SLICE_START_PTR(s)));
reinterpret_cast<uint8*>(GRPC_SLICE_START_PTR(s)));
::grpc::Slice slice(s, ::grpc::Slice::STEAL_REF);
::grpc::ByteBuffer buffer(&slice, 1);
// TODO(sanjay): Use Swap() when grpc version we are using is new enough.

View File

@ -38,7 +38,7 @@ grpc::ByteBuffer MakeBuffer(const string& str, int num_slices) {
const size_t per_slice = (str.size() + num_slices - 1) / num_slices;
for (size_t pos = 0; pos < str.size();) {
const size_t n = std::min(str.size() - pos, per_slice);
auto slice = gpr_slice_from_copied_buffer(&str[pos], n);
auto slice = grpc_slice_from_copied_buffer(&str[pos], n);
slices.push_back(::grpc::Slice(slice, ::grpc::Slice::STEAL_REF));
pos += n;
}

View File

@ -58,9 +58,9 @@ namespace grpc {
WorkerService::AsyncService::AsyncService() {
for (int i = 0; i < kGrpcNumWorkerMethods; ++i) {
AddMethod(new ::grpc::RpcServiceMethod(
AddMethod(new ::grpc::internal::RpcServiceMethod(
GrpcWorkerMethodName(static_cast<GrpcWorkerMethod>(i)),
::grpc::RpcMethod::NORMAL_RPC, nullptr));
::grpc::internal::RpcMethod::NORMAL_RPC, nullptr));
::grpc::Service::MarkMethodAsync(i);
}
}

View File

@ -130,7 +130,7 @@ namespace grpc {
// definition in "//tensorflow/core/protobuf/worker_service.proto",
// and the gRPC generated stub and service classes.
// See the proto file for the definition of methods and messages.
class WorkerService GRPC_FINAL {
class WorkerService final {
public:
class AsyncService : public ::grpc::Service {
public:

View File

@ -1,7 +1,7 @@
# Platform-specific build configurations.
load("@protobuf//:protobuf.bzl", "cc_proto_library")
load("@protobuf//:protobuf.bzl", "py_proto_library")
load("@protobuf_archive//:protobuf.bzl", "cc_proto_library")
load("@protobuf_archive//:protobuf.bzl", "py_proto_library")
load("//tensorflow:tensorflow.bzl", "if_not_mobile")
load("//tensorflow:tensorflow.bzl", "if_not_windows")
@ -44,15 +44,15 @@ def tf_proto_library_cc(name, srcs = [], has_services = None,
cc_proto_library(
name = name + "_cc",
srcs = srcs,
deps = tf_deps(protodeps, "_cc") + ["@protobuf//:cc_wkt_protos"],
cc_libs = cc_libs + ["@protobuf//:protobuf"],
deps = tf_deps(protodeps, "_cc") + ["@protobuf_archive//:cc_wkt_protos"],
cc_libs = cc_libs + ["@protobuf_archive//:protobuf"],
copts = if_not_windows([
"-Wno-unknown-warning-option",
"-Wno-unused-but-set-variable",
"-Wno-sign-compare",
]),
protoc = "@protobuf//:protoc",
default_runtime = "@protobuf//:protobuf",
protoc = "@protobuf_archive//:protoc",
default_runtime = "@protobuf_archive//:protobuf",
use_grpc_plugin = use_grpc_plugin,
testonly = testonly,
visibility = visibility,
@ -65,9 +65,9 @@ def tf_proto_library_py(name, srcs=[], protodeps=[], deps=[], visibility=[],
name = name + "_py",
srcs = srcs,
srcs_version = srcs_version,
deps = deps + tf_deps(protodeps, "_py") + ["@protobuf//:protobuf_python"],
protoc = "@protobuf//:protoc",
default_runtime = "@protobuf//:protobuf_python",
deps = deps + tf_deps(protodeps, "_py") + ["@protobuf_archive//:protobuf_python"],
protoc = "@protobuf_archive//:protoc",
default_runtime = "@protobuf_archive//:protobuf_python",
visibility = visibility,
testonly = testonly,
)

View File

@ -24,7 +24,7 @@ template <class ResponseMessage, class RequestMessage>
::grpc::ClientAsyncResponseReader<ResponseMessage>*
CreateClientAsyncResponseReader(::grpc::ChannelInterface* channel,
::grpc::CompletionQueue* cq,
const ::grpc::RpcMethod& method,
const ::grpc::internal::RpcMethod& method,
::grpc::ClientContext* context,
const RequestMessage& request) {
return new ::grpc::ClientAsyncResponseReader<ResponseMessage>(

View File

@ -38,7 +38,7 @@ template <class ResponseMessage, class RequestMessage>
::grpc::ClientAsyncResponseReader<ResponseMessage>*
CreateClientAsyncResponseReader(::grpc::ChannelInterface* channel,
::grpc::CompletionQueue* cq,
const ::grpc::RpcMethod& method,
const ::grpc::internal::RpcMethod& method,
::grpc::ClientContext* context,
const RequestMessage& request);

View File

@ -36,7 +36,7 @@ then
then
echo "Protocol buffer compiler protoc not found in PATH or in ${PROTOC}"
echo "Perhaps build it using:"
echo "bazel build --config opt @protobuf//:protoc"
echo "bazel build --config opt @protobuf_archive//:protoc"
exit 1
fi
PROTOC=$PATH_PROTOC

View File

@ -277,7 +277,7 @@ cc_binary(
linkshared = 1,
deps = [
"//tensorflow/core:framework_headers_lib",
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
],
)
@ -2518,7 +2518,7 @@ py_library(
deps = [
"//third_party/py/numpy",
"@org_python_pypi_backports_weakref",
"@protobuf//:protobuf_python",
"@protobuf_archive//:protobuf_python",
"@six_archive//:six",
],
)

View File

@ -868,7 +868,7 @@ def cc_header_only_library(name, deps=[], **kwargs):
def tf_custom_op_library_additional_deps():
return [
"@protobuf//:protobuf_headers",
"@protobuf_archive//:protobuf_headers",
clean_dep("//third_party/eigen3"),
clean_dep("//tensorflow/core:framework_headers_lib"),
]

View File

@ -95,6 +95,10 @@
#
# This script can be used by Jenkins parameterized / matrix builds.
# TODO(jhseu): Temporary for the gRPC pull request due to the
# protobuf -> protobuf_archive rename. Remove later.
TF_BUILD_BAZEL_CLEAN=1
# Helper function: Convert to lower case
to_lower () {
echo "$1" | tr '[:upper:]' '[:lower:]'

View File

@ -35,6 +35,7 @@ export TF_CUDA_COMPUTE_CAPABILITIES=3.7
$PYTHON_BIN_PATH configure.py
bazel clean
# Run bazel test command. Double test timeouts to avoid flakes.
bazel test --config=cuda --test_tag_filters=-no_gpu,-benchmark-test -k \
--jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \

View File

@ -102,7 +102,7 @@ genrule(
"@lmdb//:LICENSE",
"@local_config_sycl//sycl:LICENSE.text",
"@png_archive//:LICENSE",
"@protobuf//:LICENSE",
"@protobuf_archive//:LICENSE",
"@snappy//:COPYING",
"@zlib_archive//:zlib.h",
] + if_mkl([
@ -136,7 +136,7 @@ genrule(
"@lmdb//:LICENSE",
"@local_config_sycl//sycl:LICENSE.text",
"@png_archive//:LICENSE",
"@protobuf//:LICENSE",
"@protobuf_archive//:LICENSE",
"@snappy//:COPYING",
"@zlib_archive//:zlib.h",
] + if_mkl([

View File

@ -119,9 +119,9 @@ filegroup(
"@libxsmm_archive//:LICENSE",
"@lmdb//:LICENSE",
"@local_config_sycl//sycl:LICENSE.text",
"@nanopb_git//:LICENSE.txt",
"@grpc//third_party/nanopb:LICENSE.txt",
"@png_archive//:LICENSE",
"@protobuf//:LICENSE",
"@protobuf_archive//:LICENSE",
"@six_archive//:LICENSE",
"@snappy//:COPYING",
"@zlib_archive//:zlib.h",

View File

@ -130,7 +130,7 @@ function main() {
mkdir -p ${TMPDIR}/google
mkdir -p ${TMPDIR}/third_party
pushd ${RUNFILES%org_tensorflow}
for header in $(find protobuf -name \*.h); do
for header in $(find protobuf_archive -name \*.h); do
mkdir -p "${TMPDIR}/google/$(dirname ${header})"
cp "$header" "${TMPDIR}/google/$(dirname ${header})/"
done

View File

@ -113,7 +113,7 @@ class InstallHeaders(Command):
install_dir = os.path.join(self.install_dir, os.path.dirname(header))
# Get rid of some extra intervening directories so we can have fewer
# directories for -I
install_dir = re.sub('/google/protobuf/src', '', install_dir)
install_dir = re.sub('/google/protobuf_archive/src', '', install_dir)
# Copy eigen code into tensorflow/include.
# A symlink would do, but the wheel file that gets created ignores
@ -164,7 +164,7 @@ else:
headers = (list(find_files('*.h', 'tensorflow/core')) +
list(find_files('*.h', 'tensorflow/stream_executor')) +
list(find_files('*.h', 'google/protobuf/src')) +
list(find_files('*.h', 'google/protobuf_archive/src')) +
list(find_files('*', 'third_party/eigen3')) +
list(find_files('*', 'external/eigen_archive')))

View File

@ -358,7 +358,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
)
patched_http_archive(
name = "protobuf",
name = "protobuf_archive",
urls = [
"https://github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
"http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz",
@ -372,6 +372,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
patch_file = str(Label("//third_party/protobuf:add_noinlines.patch")),
)
native.bind(
name = "protobuf",
actual = "@protobuf_archive//:protobuf",
)
# We need to import the protobuf library under the names com_google_protobuf
# and com_google_protobuf_cc to enable proto_library support in bazel.
# Unfortunately there is no way to alias http_archives at the moment.
@ -470,23 +475,30 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
# to point to the protobuf's compiler library.
native.bind(
name = "protobuf_clib",
actual = "@protobuf//:protoc_lib",
actual = "@protobuf_archive//:protoc_lib",
)
native.bind(
name = "protobuf_compiler",
actual = "@protobuf//:protoc_lib",
name = "libssl",
actual = "@boringssl//:ssl",
)
native.new_http_archive(
# gRPC has includes directly from their third_party path for nanopb, so we
# must depend on their version of it.
native.bind(
name = "nanopb",
actual = "@grpc//third_party/nanopb:nanopb",
)
patched_http_archive(
name = "grpc",
urls = [
"http://mirror.bazel.build/github.com/grpc/grpc/archive/d7ff4ff40071d2b486a052183e3e9f9382afb745.tar.gz",
"https://github.com/grpc/grpc/archive/d7ff4ff40071d2b486a052183e3e9f9382afb745.tar.gz",
"http://mirror.bazel.build/github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz",
"https://github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz",
],
sha256 = "a15f352436ab92c521b1ac11e729e155ace38d0856380cf25048c5d1d9ba8e31",
strip_prefix = "grpc-d7ff4ff40071d2b486a052183e3e9f9382afb745",
build_file = str(Label("//third_party:grpc.BUILD")),
sha256 = "2004635e6a078acfac8ffa71738397796be4f8fb72f572cc44ecee5d99511d9f",
strip_prefix = "grpc-781fd6f6ea03645a520cd5c675da67ab61f87e4b",
patch_file = str(Label("//third_party/grpc:grpc.patch")),
)
# protobuf expects //external:grpc_cpp_plugin to point to grpc's
@ -566,22 +578,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
patch_file = str(Label("//third_party/boringssl:add_boringssl_s390x.patch")),
)
native.new_http_archive(
name = "nanopb_git",
urls = [
"http://mirror.bazel.build/github.com/nanopb/nanopb/archive/1251fa1065afc0d62f635e0f63fec8276e14e13c.tar.gz",
"https://github.com/nanopb/nanopb/archive/1251fa1065afc0d62f635e0f63fec8276e14e13c.tar.gz",
],
sha256 = "ab1455c8edff855f4f55b68480991559e51c11e7dab060bbab7cffb12dd3af33",
strip_prefix = "nanopb-1251fa1065afc0d62f635e0f63fec8276e14e13c",
build_file = str(Label("//third_party:nanopb.BUILD")),
)
native.bind(
name = "nanopb",
actual = "@nanopb_git//:nanopb",
)
native.new_http_archive(
name = "zlib_archive",
urls = [

2478
third_party/grpc.BUILD vendored

File diff suppressed because it is too large Load Diff

0
third_party/grpc/BUILD vendored Normal file
View File

76
third_party/grpc/grpc.patch vendored Normal file
View File

@ -0,0 +1,76 @@
diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl
index f793cae56d..0295adb8ab 100644
--- a/bazel/grpc_build_system.bzl
+++ b/bazel/grpc_build_system.bzl
@@ -80,7 +80,7 @@ def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data
linkopts = ["-pthread"],
)
-def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False):
+def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], data = [], language = "C++", testonly = False, linkshared = False, linkopts = []):
copts = []
if language.upper() == "C":
copts = ["-std=c99"]
@@ -93,7 +93,7 @@ def grpc_cc_binary(name, srcs = [], deps = [], external_deps = [], args = [], da
linkshared = linkshared,
deps = deps + ["//external:" + dep for dep in external_deps],
copts = copts,
- linkopts = ["-pthread"],
+ linkopts = ["-pthread"] + linkopts,
)
def grpc_generate_one_off_targets():
diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
index 7eb599d81a..4cc2e30af4 100644
--- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
+++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.c
@@ -28,18 +28,12 @@ extern void grpc_client_channel_init(void);
extern void grpc_client_channel_shutdown(void);
extern void grpc_inproc_plugin_init(void);
extern void grpc_inproc_plugin_shutdown(void);
-extern void grpc_resolver_dns_ares_init(void);
-extern void grpc_resolver_dns_ares_shutdown(void);
extern void grpc_resolver_dns_native_init(void);
extern void grpc_resolver_dns_native_shutdown(void);
extern void grpc_resolver_sockaddr_init(void);
extern void grpc_resolver_sockaddr_shutdown(void);
-extern void grpc_resolver_fake_init(void);
-extern void grpc_resolver_fake_shutdown(void);
extern void grpc_load_reporting_plugin_init(void);
extern void grpc_load_reporting_plugin_shutdown(void);
-extern void grpc_lb_policy_grpclb_init(void);
-extern void grpc_lb_policy_grpclb_shutdown(void);
extern void grpc_lb_policy_pick_first_init(void);
extern void grpc_lb_policy_pick_first_shutdown(void);
extern void grpc_lb_policy_round_robin_init(void);
@@ -64,18 +58,12 @@ void grpc_register_built_in_plugins(void) {
grpc_client_channel_shutdown);
grpc_register_plugin(grpc_inproc_plugin_init,
grpc_inproc_plugin_shutdown);
- grpc_register_plugin(grpc_resolver_dns_ares_init,
- grpc_resolver_dns_ares_shutdown);
grpc_register_plugin(grpc_resolver_dns_native_init,
grpc_resolver_dns_native_shutdown);
grpc_register_plugin(grpc_resolver_sockaddr_init,
grpc_resolver_sockaddr_shutdown);
- grpc_register_plugin(grpc_resolver_fake_init,
- grpc_resolver_fake_shutdown);
grpc_register_plugin(grpc_load_reporting_plugin_init,
grpc_load_reporting_plugin_shutdown);
- grpc_register_plugin(grpc_lb_policy_grpclb_init,
- grpc_lb_policy_grpclb_shutdown);
grpc_register_plugin(grpc_lb_policy_pick_first_init,
grpc_lb_policy_pick_first_shutdown);
grpc_register_plugin(grpc_lb_policy_round_robin_init,
diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD
index 33240f6f69..d2e1f67f06 100644
--- a/test/cpp/util/BUILD
+++ b/test/cpp/util/BUILD
@@ -29,6 +29,7 @@ package(
grpc_cc_binary(
name = "testso.so",
srcs = [],
+ linkopts = ['-Wl,--no-undefined'],
linkshared = 1,
deps = ["//:grpc++_unsecure"],
)

View File

@ -1,23 +0,0 @@
# Description:
# Nanopb, a tiny ANSI C protobuf implementation for use on embedded devices.
licenses(["notice"]) # zlib license
exports_files(["LICENSE.txt"])
cc_library(
name = "nanopb",
srcs = [
"pb_common.c",
"pb_decode.c",
"pb_encode.c",
],
hdrs = [
"pb.h",
"pb_common.h",
"pb_decode.h",
"pb_encode.h",
],
includes = ["."],
visibility = ["//visibility:public"],
)

View File

@ -4,15 +4,15 @@ package(
licenses(["notice"]) # MIT
load("@protobuf//:protobuf.bzl", "py_proto_library")
load("@protobuf_archive//:protobuf.bzl", "py_proto_library")
exports_files(["pprof/LICENSE"])
py_proto_library(
name = "pprof_proto_py",
srcs = ["proto/profile.proto"],
default_runtime = "@protobuf//:protobuf_python",
protoc = "@protobuf//:protoc",
default_runtime = "@protobuf_archive//:protobuf_python",
protoc = "@protobuf_archive//:protoc",
srcs_version = "PY2AND3",
deps = ["@protobuf//:protobuf_python"],
deps = ["@protobuf_archive//:protobuf_python"],
)