From 25251f057c476df16451599aa34a6283b9e45209 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Fri, 31 Jan 2020 12:47:04 -0800 Subject: [PATCH] Fix go proto handling --- tensorflow/core/protobuf/autotuning.proto | 2 ++ tensorflow/core/protobuf/bfc_memory_map.proto | 2 ++ .../core/protobuf/conv_autotuning.proto | 2 ++ tensorflow/core/protobuf/debug_event.proto | 2 ++ tensorflow/core/protobuf/device_filters.proto | 2 ++ tensorflow/core/protobuf/eager_service.proto | 2 ++ .../core/protobuf/graph_debug_info.proto | 2 ++ .../core/protobuf/remote_tensor_handle.proto | 2 ++ tensorflow/core/protobuf/replay_log.proto | 2 ++ .../core/protobuf/saved_object_graph.proto | 2 ++ tensorflow/core/protobuf/struct.proto | 2 ++ tensorflow/core/protobuf/trace_events.proto | 2 ++ .../protobuf/trackable_object_graph.proto | 2 ++ .../core/protobuf/transport_options.proto | 2 ++ tensorflow/go/genop/generate.sh | 14 +++++--- tensorflow/go/saved_model.go | 2 +- tensorflow/go/signature.go | 2 +- tensorflow/go/signature_test.go | 35 +++++++++---------- tensorflow/stream_executor/dnn.proto | 2 ++ 19 files changed, 58 insertions(+), 25 deletions(-) diff --git a/tensorflow/core/protobuf/autotuning.proto b/tensorflow/core/protobuf/autotuning.proto index f43dbbeac5e..1cecc6faff0 100644 --- a/tensorflow/core/protobuf/autotuning.proto +++ b/tensorflow/core/protobuf/autotuning.proto @@ -10,6 +10,8 @@ package tensorflow; import "google/protobuf/any.proto"; import "google/protobuf/duration.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + message CudnnVersion { int32 major = 1; int32 minor = 2; diff --git a/tensorflow/core/protobuf/bfc_memory_map.proto b/tensorflow/core/protobuf/bfc_memory_map.proto index 202903d2bf4..5ac9bdb5e55 100644 --- a/tensorflow/core/protobuf/bfc_memory_map.proto +++ b/tensorflow/core/protobuf/bfc_memory_map.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package tensorflow; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // Some of the data from AllocatorStats message MemAllocatorStats { int64 num_allocs = 1; diff --git a/tensorflow/core/protobuf/conv_autotuning.proto b/tensorflow/core/protobuf/conv_autotuning.proto index 5616299a6de..e49be02f5f7 100644 --- a/tensorflow/core/protobuf/conv_autotuning.proto +++ b/tensorflow/core/protobuf/conv_autotuning.proto @@ -6,6 +6,8 @@ package tensorflow; import "tensorflow/stream_executor/dnn.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // A convolution. Currently it's only used for logging. In the future, we may // want to use it in the API as well. message ConvolutionProto { diff --git a/tensorflow/core/protobuf/debug_event.proto b/tensorflow/core/protobuf/debug_event.proto index badd518fa69..4b34864361c 100644 --- a/tensorflow/core/protobuf/debug_event.proto +++ b/tensorflow/core/protobuf/debug_event.proto @@ -10,6 +10,8 @@ option java_outer_classname = "DebugEventProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.util"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // Available modes for extracting debugging information from a Tensor. // TODO(cais): Document the detailed column names and semantics in a separate // markdown file once the implementation settles. diff --git a/tensorflow/core/protobuf/device_filters.proto b/tensorflow/core/protobuf/device_filters.proto index 0aa38379a2a..bbfa0af11f7 100644 --- a/tensorflow/core/protobuf/device_filters.proto +++ b/tensorflow/core/protobuf/device_filters.proto @@ -22,6 +22,8 @@ option java_outer_classname = "DeviceFiltersProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.distruntime"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // This file contains protos to be used when defining a TensorFlow // cluster. // diff --git a/tensorflow/core/protobuf/eager_service.proto b/tensorflow/core/protobuf/eager_service.proto index 967df44d3dc..e321d11ae5b 100644 --- a/tensorflow/core/protobuf/eager_service.proto +++ b/tensorflow/core/protobuf/eager_service.proto @@ -11,6 +11,8 @@ import "tensorflow/core/framework/versions.proto"; import "tensorflow/core/protobuf/remote_tensor_handle.proto"; import "tensorflow/core/protobuf/tensorflow_server.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // A proto representation of an eager operation. message Operation { // A unique identifier for the operation. Set by the client so that the client diff --git a/tensorflow/core/protobuf/graph_debug_info.proto b/tensorflow/core/protobuf/graph_debug_info.proto index a3e3b18d72a..dddcc0ef341 100644 --- a/tensorflow/core/protobuf/graph_debug_info.proto +++ b/tensorflow/core/protobuf/graph_debug_info.proto @@ -7,6 +7,8 @@ option java_outer_classname = "GraphDebugInfoProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + message GraphDebugInfo { // This represents a file/line location in the source code. message FileLineCol { diff --git a/tensorflow/core/protobuf/remote_tensor_handle.proto b/tensorflow/core/protobuf/remote_tensor_handle.proto index b8db44857c7..6148d602a6d 100644 --- a/tensorflow/core/protobuf/remote_tensor_handle.proto +++ b/tensorflow/core/protobuf/remote_tensor_handle.proto @@ -10,6 +10,8 @@ option java_outer_classname = "RemoteTensorHandleProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + message ResourceDtypeAndShape { DataType dtype = 1; TensorShapeProto shape = 2; diff --git a/tensorflow/core/protobuf/replay_log.proto b/tensorflow/core/protobuf/replay_log.proto index 5506ec0c8ea..80aa6ee85bb 100644 --- a/tensorflow/core/protobuf/replay_log.proto +++ b/tensorflow/core/protobuf/replay_log.proto @@ -6,6 +6,8 @@ import "tensorflow/core/protobuf/master.proto"; option cc_enable_arenas = true; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // Records the creation of a new replay session. We record the device listing // here to capture the state of the cluster. message NewReplaySession { diff --git a/tensorflow/core/protobuf/saved_object_graph.proto b/tensorflow/core/protobuf/saved_object_graph.proto index 20074f31258..d7577067ff9 100644 --- a/tensorflow/core/protobuf/saved_object_graph.proto +++ b/tensorflow/core/protobuf/saved_object_graph.proto @@ -11,6 +11,8 @@ option cc_enable_arenas = true; package tensorflow; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // A SavedObjectGraph is part of object-based SavedModels in TF 2.0. It // describes the directed graph of Python objects (or equivalent in other // languages) that make up a model, with nodes[0] at the root. diff --git a/tensorflow/core/protobuf/struct.proto b/tensorflow/core/protobuf/struct.proto index 8983db02eeb..0508640f814 100644 --- a/tensorflow/core/protobuf/struct.proto +++ b/tensorflow/core/protobuf/struct.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package tensorflow; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + import "tensorflow/core/framework/tensor_shape.proto"; import "tensorflow/core/framework/types.proto"; diff --git a/tensorflow/core/protobuf/trace_events.proto b/tensorflow/core/protobuf/trace_events.proto index 76b7300aeab..84cbea5bacf 100644 --- a/tensorflow/core/protobuf/trace_events.proto +++ b/tensorflow/core/protobuf/trace_events.proto @@ -7,6 +7,8 @@ option java_outer_classname = "TraceEventsProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // A 'Trace' contains metadata for the individual traces of a system. message Trace { // The devices that this trace has information about. Maps from device_id to diff --git a/tensorflow/core/protobuf/trackable_object_graph.proto b/tensorflow/core/protobuf/trackable_object_graph.proto index 02d852e6f3d..6b1a3b9df4d 100644 --- a/tensorflow/core/protobuf/trackable_object_graph.proto +++ b/tensorflow/core/protobuf/trackable_object_graph.proto @@ -4,6 +4,8 @@ option cc_enable_arenas = true; package tensorflow; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // A TensorBundle addition which saves extra information about the objects which // own variables, allowing for more robust checkpoint loading into modified // programs. diff --git a/tensorflow/core/protobuf/transport_options.proto b/tensorflow/core/protobuf/transport_options.proto index 1d32475e9b9..fb202d1067e 100644 --- a/tensorflow/core/protobuf/transport_options.proto +++ b/tensorflow/core/protobuf/transport_options.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package tensorflow; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; + // Extra data needed on a non-RDMA RecvBufResponse. message RecvBufRespExtra { repeated bytes tensor_content = 1; diff --git a/tensorflow/go/genop/generate.sh b/tensorflow/go/genop/generate.sh index 446ddbe35a5..18bee11da5a 100644 --- a/tensorflow/go/genop/generate.sh +++ b/tensorflow/go/genop/generate.sh @@ -51,8 +51,12 @@ fi # Ensure that protoc-gen-go is available in $PATH # Since ${PROTOC} will require it. export PATH=$PATH:${GOPATH}/bin -mkdir -p ./internal/proto -${PROTOC} \ - -I ${TF_DIR} \ - --go_out=./internal/proto \ - ${TF_DIR}/tensorflow/core/framework/*.proto +mkdir -p ../vendor +for FILE in ${TF_DIR}/tensorflow/core/framework/*.proto \ + ${TF_DIR}/tensorflow/core/protobuf/*.proto \ + ${TF_DIR}/tensorflow/stream_executor/*.proto; do + ${PROTOC} \ + -I ${TF_DIR} \ + --go_out=../vendor \ + $FILE +done diff --git a/tensorflow/go/saved_model.go b/tensorflow/go/saved_model.go index 90fe3655585..fdcc124b200 100644 --- a/tensorflow/go/saved_model.go +++ b/tensorflow/go/saved_model.go @@ -22,7 +22,7 @@ import ( "github.com/golang/protobuf/proto" - tfpb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core" + tfpb "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf" ) // #include diff --git a/tensorflow/go/signature.go b/tensorflow/go/signature.go index 67d62577125..868c09e2a22 100644 --- a/tensorflow/go/signature.go +++ b/tensorflow/go/signature.go @@ -16,7 +16,7 @@ limitations under the License. package tensorflow -import tfpb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core" +import tfpb "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf" // #include "tensorflow/c/c_api.h" import "C" diff --git a/tensorflow/go/signature_test.go b/tensorflow/go/signature_test.go index 7988347ed17..8c3fb2d9d9d 100644 --- a/tensorflow/go/signature_test.go +++ b/tensorflow/go/signature_test.go @@ -20,9 +20,8 @@ import ( "fmt" "testing" - tspb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_shape_go_proto" - typb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core/framework/types_go_proto" - tfpb "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal/proto/github.com/tensorflow/tensorflow/tensorflow/go/core" + frameworkpb "github.com/tensorflow/tensorflow/tensorflow/go/core/framework" + tfpb "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf" ) func TestSignatureFromProto(t *testing.T) { @@ -32,9 +31,9 @@ func TestSignatureFromProto(t *testing.T) { Encoding: &tfpb.TensorInfo_Name{ Name: "tensor_1", }, - Dtype: typb.DataType_DT_INT8, - TensorShape: &tspb.TensorShapeProto{ - Dim: []*tspb.TensorShapeProto_Dim{ + Dtype: frameworkpb.DataType_DT_INT8, + TensorShape: &frameworkpb.TensorShapeProto{ + Dim: []*frameworkpb.TensorShapeProto_Dim{ {Size: 1}, {Size: 2}, {Size: 3}, @@ -45,9 +44,9 @@ func TestSignatureFromProto(t *testing.T) { Encoding: &tfpb.TensorInfo_Name{ Name: "tensor_2", }, - Dtype: typb.DataType_DT_FLOAT, - TensorShape: &tspb.TensorShapeProto{ - Dim: []*tspb.TensorShapeProto_Dim{ + Dtype: frameworkpb.DataType_DT_FLOAT, + TensorShape: &frameworkpb.TensorShapeProto{ + Dim: []*frameworkpb.TensorShapeProto_Dim{ {Size: 4}, {Size: 5}, {Size: 6}, @@ -60,9 +59,9 @@ func TestSignatureFromProto(t *testing.T) { Encoding: &tfpb.TensorInfo_Name{ Name: "tensor_3", }, - Dtype: typb.DataType_DT_STRING, - TensorShape: &tspb.TensorShapeProto{ - Dim: []*tspb.TensorShapeProto_Dim{ + Dtype: frameworkpb.DataType_DT_STRING, + TensorShape: &frameworkpb.TensorShapeProto{ + Dim: []*frameworkpb.TensorShapeProto_Dim{ {Size: 1}, {Size: 2}, {Size: 3}, @@ -73,9 +72,9 @@ func TestSignatureFromProto(t *testing.T) { Encoding: &tfpb.TensorInfo_Name{ Name: "tensor_4", }, - Dtype: typb.DataType_DT_BOOL, - TensorShape: &tspb.TensorShapeProto{ - Dim: []*tspb.TensorShapeProto_Dim{ + Dtype: frameworkpb.DataType_DT_BOOL, + TensorShape: &frameworkpb.TensorShapeProto{ + Dim: []*frameworkpb.TensorShapeProto_Dim{ {Size: 4}, {Size: 5}, {Size: 6}, @@ -144,9 +143,9 @@ func TestTensorInfoFromProto(t *testing.T) { Encoding: &tfpb.TensorInfo_Name{ Name: "tensor", }, - Dtype: typb.DataType_DT_INT8, - TensorShape: &tspb.TensorShapeProto{ - Dim: []*tspb.TensorShapeProto_Dim{ + Dtype: frameworkpb.DataType_DT_INT8, + TensorShape: &frameworkpb.TensorShapeProto{ + Dim: []*frameworkpb.TensorShapeProto_Dim{ {Size: 1}, {Size: 2}, {Size: 3}, diff --git a/tensorflow/stream_executor/dnn.proto b/tensorflow/stream_executor/dnn.proto index fb6bda9d317..4d09e615e7d 100644 --- a/tensorflow/stream_executor/dnn.proto +++ b/tensorflow/stream_executor/dnn.proto @@ -3,6 +3,8 @@ syntax = "proto3"; package stream_executor.dnn; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/stream_executor"; + // Specifies the data type used by an operation. enum DataType { kFloat = 0;