From d6f107761459dfdf8773a148e11193a3512a51a6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 6 Sep 2018 17:11:37 -0700 Subject: [PATCH] Automated rollback of commit 24787842adfefe35f5a520313d775b14c29f143a PiperOrigin-RevId: 211895566 --- .../compiler/aot/embedded_protocol_buffers.h | 1 + tensorflow/compiler/aot/tfcompile_main.cc | 6 +- .../jit/mark_for_compilation_pass_test.cc | 2 +- tensorflow/compiler/jit/xla_cluster_util.h | 1 + tensorflow/compiler/jit/xla_device_context.cc | 6 +- tensorflow/compiler/jit/xla_device_context.h | 8 +- tensorflow/compiler/tf2xla/BUILD | 1 + .../tf2xla/resource_operation_table.cc | 18 +-- tensorflow/compiler/tf2xla/tf2xla_util.h | 1 + tensorflow/compiler/tf2xla/xla_op_kernel.cc | 11 +- tensorflow/compiler/tf2xla/xla_op_registry.h | 1 + .../compiler/xla/packed_literal_reader.cc | 5 +- .../contrib/makefile/proto_text_cc_files.txt | 1 + tensorflow/core/lib/core/stringpiece.cc | 54 ++++++++ tensorflow/core/lib/core/stringpiece.h | 117 +++++++++++++++++- tensorflow/core/lib/strings/strcat.h | 3 - 16 files changed, 206 insertions(+), 30 deletions(-) create mode 100644 tensorflow/core/lib/core/stringpiece.cc diff --git a/tensorflow/compiler/aot/embedded_protocol_buffers.h b/tensorflow/compiler/aot/embedded_protocol_buffers.h index cf5c04ac4bd..bd270045e38 100644 --- a/tensorflow/compiler/aot/embedded_protocol_buffers.h +++ b/tensorflow/compiler/aot/embedded_protocol_buffers.h @@ -20,6 +20,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_AOT_EMBEDDED_PROTOCOL_BUFFERS_H_ #define TENSORFLOW_COMPILER_AOT_EMBEDDED_PROTOCOL_BUFFERS_H_ +#include "absl/strings/string_view.h" #include "absl/types/span.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/core/platform/protobuf.h" diff --git a/tensorflow/compiler/aot/tfcompile_main.cc b/tensorflow/compiler/aot/tfcompile_main.cc index b95b063348c..1c9d30d7b08 100644 --- a/tensorflow/compiler/aot/tfcompile_main.cc +++ b/tensorflow/compiler/aot/tfcompile_main.cc @@ -35,6 +35,7 @@ limitations under the License. #include "tensorflow/core/graph/graph.h" #include "tensorflow/core/graph/tensor_id.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/init_main.h" @@ -92,9 +93,8 @@ Status Main(const MainFlags& flags) { // Write output files. Env* env = Env::Default(); const std::vector& obj = compile_result.aot->object_file_data(); - TF_RETURN_IF_ERROR( - WriteStringToFile(env, flags.out_function_object, - absl::string_view(obj.data(), obj.size()))); + TF_RETURN_IF_ERROR(WriteStringToFile(env, flags.out_function_object, + StringPiece(obj.data(), obj.size()))); CodegenOpts codegen_opts; codegen_opts.gen_name_to_index = flags.gen_name_to_index; codegen_opts.gen_program_shape = flags.gen_program_shape; diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index 9473ac0a4c6..807ab51fd3c 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -633,7 +633,7 @@ TEST(XlaCompilationTest, IllegalCycle_UsefulErrorMessage) { std::unique_ptr graph(new Graph(OpRegistry::Global())); Scope root = Scope::NewRootScope().ExitOnError(); { - auto BuildNoopNode = [](absl::string_view name, Graph* graph) { + auto BuildNoopNode = [](StringPiece name, Graph* graph) { NodeDefBuilder builder(name, "NoOp"); NodeDef def; TF_CHECK_OK(builder.Finalize(&def)); diff --git a/tensorflow/compiler/jit/xla_cluster_util.h b/tensorflow/compiler/jit/xla_cluster_util.h index 17ae510a0eb..debd9038c7d 100644 --- a/tensorflow/compiler/jit/xla_cluster_util.h +++ b/tensorflow/compiler/jit/xla_cluster_util.h @@ -18,6 +18,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_JIT_XLA_CLUSTER_UTIL_H_ #define TENSORFLOW_COMPILER_JIT_XLA_CLUSTER_UTIL_H_ +#include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "tensorflow/compiler/jit/graphcycles/graphcycles.h" #include "tensorflow/core/graph/algorithm.h" diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc index af83c792e5e..6d4160a9681 100644 --- a/tensorflow/compiler/jit/xla_device_context.cc +++ b/tensorflow/compiler/jit/xla_device_context.cc @@ -339,11 +339,11 @@ void XlaDeviceContext::CopyCPUTensorToDevice(const Tensor* cpu_tensor, } void XlaDeviceContext::CopyDeviceTensorToCPU(const Tensor* device_tensor, - absl::string_view tensor_name, + StringPiece tensor_name, Device* device, Tensor* cpu_tensor, StatusCallback done) { - manager_.CopyDeviceTensorToCPU(device_tensor, tensor_name, device, cpu_tensor, - done); + manager_.CopyDeviceTensorToCPU(device_tensor, absl::string_view(tensor_name), + device, cpu_tensor, done); } void XlaDeviceContext::CopyDeviceTensorToDevice(const Tensor& src_tensor, diff --git a/tensorflow/compiler/jit/xla_device_context.h b/tensorflow/compiler/jit/xla_device_context.h index df824212948..1effd6628f8 100644 --- a/tensorflow/compiler/jit/xla_device_context.h +++ b/tensorflow/compiler/jit/xla_device_context.h @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/core/framework/allocator.h" #include "tensorflow/core/framework/device_base.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/stringpiece.h" namespace tensorflow { @@ -110,9 +111,12 @@ class XlaDeviceContext : public DeviceContext { void CopyCPUTensorToDevice(const Tensor* cpu_tensor, Device* device, Tensor* device_tensor, StatusCallback done) const override; + // TODO(rlahaye): Replace StringPiece with absl::string_view when the + // StringPiece->absl::string_view change is rolled forward. void CopyDeviceTensorToCPU(const Tensor* device_tensor, - absl::string_view tensor_name, Device* device, - Tensor* cpu_tensor, StatusCallback done) override; + StringPiece tensor_name, // non-ABSL OK + Device* device, Tensor* cpu_tensor, + StatusCallback done) override; void CopyDeviceTensorToDevice(const Tensor& src_tensor, Tensor* dst_tensor, const StatusCallback& done); diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index 22be7f048fc..95004534b9b 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -214,6 +214,7 @@ cc_library( "//tensorflow/core:protos_all_cc", "//tensorflow/core:stream_executor_no_cuda", "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", "@com_google_absl//absl/types:span", ], alwayslink = 1, diff --git a/tensorflow/compiler/tf2xla/resource_operation_table.cc b/tensorflow/compiler/tf2xla/resource_operation_table.cc index 20f2ce29197..92577b5bc82 100644 --- a/tensorflow/compiler/tf2xla/resource_operation_table.cc +++ b/tensorflow/compiler/tf2xla/resource_operation_table.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/resource_operation_table.h" #include "absl/algorithm/container.h" +#include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/gtl/flatmap.h" namespace tensorflow { @@ -30,11 +31,10 @@ namespace tensorflow { } } -static gtl::FlatMap* -CreateResourceOpInfoMap() { - auto* result = new gtl::FlatMap; +static gtl::FlatMap* CreateResourceOpInfoMap() { + auto* result = new gtl::FlatMap; - auto add = [&](absl::string_view op, XlaResourceOpKind op_kind, + auto add = [&](StringPiece op, XlaResourceOpKind op_kind, XlaResourceKind resource_kind) { auto insert_result = result->insert({op, XlaResourceOpInfo(op_kind, resource_kind)}); @@ -103,17 +103,17 @@ CreateResourceOpInfoMap() { return result; } -static const gtl::FlatMap& +static const gtl::FlatMap& GetStaticResourceOpInfoMap() { - static gtl::FlatMap* op_info_map = + static gtl::FlatMap* op_info_map = CreateResourceOpInfoMap(); return *op_info_map; } const XlaResourceOpInfo* GetResourceOpInfoForOp(absl::string_view op) { - const gtl::FlatMap& op_infos = + const gtl::FlatMap& op_infos = GetStaticResourceOpInfoMap(); - auto it = op_infos.find(op); + auto it = op_infos.find(StringPiece(op.data(), op.length())); return it == op_infos.end() ? nullptr : &it->second; } @@ -121,7 +121,7 @@ namespace resource_op_table_internal { std::vector GetKnownResourceOps() { std::vector result; for (const auto& p : GetStaticResourceOpInfoMap()) { - result.push_back(p.first); + result.push_back(absl::string_view(p.first)); } absl::c_sort(result); return result; diff --git a/tensorflow/compiler/tf2xla/tf2xla_util.h b/tensorflow/compiler/tf2xla/tf2xla_util.h index a29e764466c..dcddef84182 100644 --- a/tensorflow/compiler/tf2xla/tf2xla_util.h +++ b/tensorflow/compiler/tf2xla/tf2xla_util.h @@ -18,6 +18,7 @@ limitations under the License. #include +#include "absl/strings/string_view.h" #include "tensorflow/compiler/tf2xla/tf2xla.pb.h" #include "tensorflow/core/framework/graph.pb.h" #include "tensorflow/core/framework/kernel_def.pb.h" diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc index d67e50375b8..636cb71e21b 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc @@ -102,7 +102,8 @@ Status XlaOpKernelContext::ConstantInput(int index, static xla::StatusOr InputIndex(XlaOpKernelContext* context, absl::string_view name) { int start, stop; - TF_RETURN_IF_ERROR(context->op_kernel().InputRange(name, &start, &stop)); + TF_RETURN_IF_ERROR(context->op_kernel().InputRange( + StringPiece(name.data(), name.length()), &start, &stop)); if (stop != start + 1) { return errors::InvalidArgument("OpKernel used list-valued input name '", name, @@ -365,7 +366,8 @@ Status XlaOpKernelContext::InputList(absl::string_view name, std::vector* handles, std::vector* shapes) { OpInputList inputs; - TF_RETURN_IF_ERROR(context_->input_list(name, &inputs)); + TF_RETURN_IF_ERROR( + context_->input_list(StringPiece(name.data(), name.size()), &inputs)); handles->clear(); shapes->clear(); for (const Tensor& input : inputs) { @@ -378,7 +380,8 @@ Status XlaOpKernelContext::InputList(absl::string_view name, Status XlaOpKernelContext::ConstantInputList( absl::string_view name, std::vector* outputs) { int start, stop; - TF_RETURN_IF_ERROR(op_kernel().InputRange(name, &start, &stop)); + TF_RETURN_IF_ERROR(op_kernel().InputRange( + StringPiece(name.data(), name.size()), &start, &stop)); outputs->resize(stop - start); for (int i = start; i < stop; ++i) { TF_RETURN_IF_ERROR(ConstantInput(i, &(*outputs)[i])); @@ -612,7 +615,7 @@ const xla::XlaComputation* XlaOpKernelContext::GetOrCreateMul( const Tensor& XlaOpKernelContext::GetInputTensorByName(absl::string_view name) { const Tensor* tensor; - CHECK(context_->input(name, &tensor).ok()); + CHECK(context_->input(StringPiece(name.data(), name.length()), &tensor).ok()); return *tensor; } diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h index 74a4885f1f0..5d53169f689 100644 --- a/tensorflow/compiler/tf2xla/xla_op_registry.h +++ b/tensorflow/compiler/tf2xla/xla_op_registry.h @@ -22,6 +22,7 @@ limitations under the License. #include #include +#include "absl/strings/string_view.h" #include "tensorflow/core/common_runtime/device_factory.h" #include "tensorflow/core/common_runtime/local_device.h" #include "tensorflow/core/framework/device_base.h" diff --git a/tensorflow/compiler/xla/packed_literal_reader.cc b/tensorflow/compiler/xla/packed_literal_reader.cc index f9473d372bb..bddb6641495 100644 --- a/tensorflow/compiler/xla/packed_literal_reader.cc +++ b/tensorflow/compiler/xla/packed_literal_reader.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/util.h" #include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/platform/types.h" @@ -64,7 +65,7 @@ StatusOr> PackedLiteralReader::Read( absl::Span field = result->data(); char* data = absl::bit_cast(field.data()); uint64 bytes = elements * sizeof(float); - absl::string_view sp; + tensorflow::StringPiece sp; auto s = file_->Read(offset_, bytes, &sp, data); offset_ += sp.size(); if (!s.ok()) { @@ -85,7 +86,7 @@ bool PackedLiteralReader::IsExhausted() const { // Try to read a single byte from offset_. If we can't, we've // exhausted the data. char single_byte[1]; - absl::string_view sp; + tensorflow::StringPiece sp; auto s = file_->Read(offset_, sizeof(single_byte), &sp, single_byte); return !s.ok(); } diff --git a/tensorflow/contrib/makefile/proto_text_cc_files.txt b/tensorflow/contrib/makefile/proto_text_cc_files.txt index 22b11f1c579..7d26429f9c3 100644 --- a/tensorflow/contrib/makefile/proto_text_cc_files.txt +++ b/tensorflow/contrib/makefile/proto_text_cc_files.txt @@ -56,6 +56,7 @@ tensorflow/core/lib/hash/hash.cc tensorflow/core/lib/hash/crc32c.cc tensorflow/core/lib/hash/crc32c_accelerate.cc tensorflow/core/lib/core/threadpool.cc +tensorflow/core/lib/core/stringpiece.cc tensorflow/core/lib/core/status.cc tensorflow/core/lib/core/coding.cc tensorflow/core/lib/core/arena.cc diff --git a/tensorflow/core/lib/core/stringpiece.cc b/tensorflow/core/lib/core/stringpiece.cc new file mode 100644 index 00000000000..4c488066e4b --- /dev/null +++ b/tensorflow/core/lib/core/stringpiece.cc @@ -0,0 +1,54 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/lib/core/stringpiece.h" + +#include +#include + +namespace tensorflow { + +std::ostream& operator<<(std::ostream& o, StringPiece piece) { + o.write(piece.data(), piece.size()); + return o; +} + +size_t StringPiece::find(char c, size_t pos) const { + if (pos >= size_) { + return npos; + } + const char* result = + reinterpret_cast(memchr(data_ + pos, c, size_ - pos)); + return result != nullptr ? result - data_ : npos; +} + +// Search range is [0..pos] inclusive. If pos == npos, search everything. +size_t StringPiece::rfind(char c, size_t pos) const { + if (size_ == 0) return npos; + for (const char* p = data_ + std::min(pos, size_ - 1); p >= data_; p--) { + if (*p == c) { + return p - data_; + } + } + return npos; +} + +StringPiece StringPiece::substr(size_t pos, size_t n) const { + if (pos > size_) pos = size_; + if (n > size_ - pos) n = size_ - pos; + return StringPiece(data_ + pos, n); +} + +} // namespace tensorflow diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h index e7b17c9b369..02dded42c14 100644 --- a/tensorflow/core/lib/core/stringpiece.h +++ b/tensorflow/core/lib/core/stringpiece.h @@ -31,13 +31,124 @@ limitations under the License. #include #include #include -#include "absl/strings/string_view.h" +#include #include "tensorflow/core/platform/types.h" namespace tensorflow { -// Deprecated: please use absl::string_view directly. -using StringPiece = absl::string_view; +class StringPiece { + public: + typedef size_t size_type; + + // Create an empty slice. + StringPiece() : data_(nullptr), size_(0) {} + + // Create a slice that refers to d[0,n-1]. + StringPiece(const char* d, size_t n) : data_(d), size_(n) {} + + // Create a slice that refers to the contents of "s" + StringPiece(const string& s) : data_(s.data()), size_(s.size()) {} + + // Create a slice that refers to s[0,strlen(s)-1] + StringPiece(const char* s) : data_(s), size_(strlen(s)) {} + + // Return a pointer to the beginning of the referenced data + const char* data() const { return data_; } + + // Return the length (in bytes) of the referenced data + size_t size() const { return size_; } + + // Return true iff the length of the referenced data is zero + bool empty() const { return size_ == 0; } + + typedef const char* const_iterator; + typedef const char* iterator; + iterator begin() const { return data_; } + iterator end() const { return data_ + size_; } + + static const size_t npos = size_type(-1); + + // Return the ith byte in the referenced data. + // REQUIRES: n < size() + char operator[](size_t n) const { + assert(n < size()); + return data_[n]; + } + + // Drop the first "n" bytes from this slice. + void remove_prefix(size_t n) { + assert(n <= size()); + data_ += n; + size_ -= n; + } + + void remove_suffix(size_t n) { + assert(size_ >= n); + size_ -= n; + } + + size_t find(char c, size_t pos = 0) const; + size_t rfind(char c, size_t pos = npos) const; + + StringPiece substr(size_t pos, size_t n = npos) const; + + // Three-way comparison. Returns value: + // < 0 iff "*this" < "b", + // == 0 iff "*this" == "b", + // > 0 iff "*this" > "b" + int compare(StringPiece b) const; + + // Converts to various kinds of strings, including `std::basic_string`. + template + explicit operator S() const { + static_assert( + std::is_same::value, + "Type mismatch: S must be a string with character type char."); + static_assert( + std::is_same, typename S::traits_type>::value, + "Type mismatch: S must be a string with traits type " + "std::char_traits."); + if (!data()) return {}; + return S(data(), size()); + } + + private: + const char* data_; + size_t size_; + + // Intentionally copyable +}; + +inline bool operator==(StringPiece x, StringPiece y) { + return ((x.size() == y.size()) && + (memcmp(x.data(), y.data(), x.size()) == 0)); +} + +inline bool operator!=(StringPiece x, StringPiece y) { return !(x == y); } + +inline bool operator<(StringPiece x, StringPiece y) { return x.compare(y) < 0; } +inline bool operator>(StringPiece x, StringPiece y) { return x.compare(y) > 0; } +inline bool operator<=(StringPiece x, StringPiece y) { + return x.compare(y) <= 0; +} +inline bool operator>=(StringPiece x, StringPiece y) { + return x.compare(y) >= 0; +} + +inline int StringPiece::compare(StringPiece b) const { + const size_t min_len = (size_ < b.size_) ? size_ : b.size_; + int r = memcmp(data_, b.data_, min_len); + if (r == 0) { + if (size_ < b.size_) + r = -1; + else if (size_ > b.size_) + r = +1; + } + return r; +} + +// allow StringPiece to be logged +extern std::ostream& operator<<(std::ostream& o, tensorflow::StringPiece piece); } // namespace tensorflow diff --git a/tensorflow/core/lib/strings/strcat.h b/tensorflow/core/lib/strings/strcat.h index a620f594476..351b6f5de39 100644 --- a/tensorflow/core/lib/strings/strcat.h +++ b/tensorflow/core/lib/strings/strcat.h @@ -124,9 +124,6 @@ class AlphaNum { AlphaNum(const StringPiece &pc) : piece_(pc) {} // NOLINT(runtime/explicit) AlphaNum(const tensorflow::string &str) // NOLINT(runtime/explicit) : piece_(str) {} - template - AlphaNum(const std::basic_string, A> &str) - : piece_(str) {} // NOLINT(runtime/explicit) StringPiece::size_type size() const { return piece_.size(); } const char *data() const { return piece_.data(); }