experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixes/updates for tensorflow::tstring migration.

Browse Source 
Addressed backsliding, several memory related bugs for tstrings, and a few overlooked migrations.

PiperOrigin-RevId: 282708800
Change-Id: I0ed40b45e7a95a99bb7fed8a6e7fe85cac0e2b71
This commit is contained in:
Dero Gharibian 2019-11-26 23:31:33 -08:00 committed by TensorFlower Gardener
parent 20968b9dca
commit bf6e90ccc6
19 changed files with 71 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tensorflow/cc/framework/cc_op_gen.cc
View File

@ -329,7 +329,7 @@ std::pair<const char*, bool> AttrTypeName(StringPiece attr_type) {
new std::unordered_map<StringPiece, std::pair<const char*, bool>,
StringPieceHasher>{
{"string", {"StringPiece", false}},
{"list(string)", {"gtl::ArraySlice<string>", true}},
{"list(string)", {"gtl::ArraySlice<::tensorflow::tstring>", true}},
{"int", {"int64", false}},
{"list(int)", {"gtl::ArraySlice<int>", true}},
{"float", {"float", false}},

4
tensorflow/compiler/tf2tensorrt/kernels/trt_engine_resource_ops_test.cc
View File

@ -127,7 +127,7 @@ TEST_F(TRTEngineResourceOpsTest, Basic) {
.Finalize(node_def()));
TF_ASSERT_OK(InitOp());
AddInputFromArray<ResourceHandle>(TensorShape({}), {handle});
AddInputFromArray<string>(TensorShape({}), {filename});
AddInputFromArray<tstring>(TensorShape({}), {filename});
TF_ASSERT_OK(RunOpKernel());
EXPECT_TRUE(rm->Lookup(container, resource_name, &resource).ok());
EXPECT_EQ(0, resource->cache_.size());
@ -168,7 +168,7 @@ TEST_F(TRTEngineResourceOpsTest, Basic) {
TF_ASSERT_OK(env->NewRandomAccessFile(filename, &file));
auto reader = absl::make_unique<io::RecordReader>(file.get());
uint64 offset = 0;
string record;
tstring record;
TF_ASSERT_OK(reader->ReadRecord(&offset, &record));
TRTEngineInstance engine_instance;
engine_instance.ParseFromString(record);

5
tensorflow/core/kernels/concat_lib_cpu.cc
View File

@ -57,10 +57,9 @@ int64 EstimateBytesPerElement(
// reshapes all the inputs to matrices), by sampling the lengths of the actual
// strings in the various tensors.
template <>
int64 EstimateBytesPerElement<std::string>(
int64 EstimateBytesPerElement<tstring>(
const std::vector<
std::unique_ptr<typename TTypes<std::string, 2>::ConstMatrix>>&
inputs) {
std::unique_ptr<typename TTypes<tstring, 2>::ConstMatrix>>& inputs) {
// randomly sample a few input strings to get a sense of the average size
// of each element
int num_samples = 0;

10
tensorflow/core/kernels/concat_op_test.cc
View File

@ -42,12 +42,12 @@ void FillTensorWithRandomValues(Tensor* t, int string_length, int64* bytes) {
}
template <>
void FillTensorWithRandomValues<std::string>(Tensor* t, int string_length,
int64* bytes) {
auto ts = t->flat<string>();
void FillTensorWithRandomValues<tstring>(Tensor* t, int string_length,
int64* bytes) {
auto ts = t->flat<tstring>();
*bytes = 0;
for (int i = 0; i < ts.size(); i++) {
ts(i) = string(string_length, 'x');
ts(i) = tstring(string_length, 'x');
*bytes += sizeof(ts(i)) + ts(i).size();
}
}
@ -99,7 +99,7 @@ BENCHMARK(BM_ConcatDim0Float)->Arg(1000)->Arg(100000)->Arg(1000000);
BENCHMARK(BM_ConcatDim1Float)->Arg(1000)->Arg(100000)->Arg(1000000);
static void BM_ConcatDim0String(int iters, int dim2, int string_length) {
ConcatHelper<string>(iters, 0, dim2, string_length);
ConcatHelper<tstring>(iters, 0, dim2, string_length);
}
BENCHMARK(BM_ConcatDim0String)

20
tensorflow/core/kernels/data/experimental/unique_dataset_op_test.cc
View File

@ -99,7 +99,7 @@ UniqueDatasetParams EmptyInputParams() {
UniqueDatasetParams StringParams() {
auto tensor_slice_dataset_params = TensorSliceDatasetParams(
/*components=*/
{CreateTensor<string>(
{CreateTensor<tstring>(
TensorShape{11, 1},
{"one", "One", "two", "three", "five", "eight", "thirteen",
"twenty-one", "eight", "eight", "thirty-four"})},
@ -164,15 +164,15 @@ std::vector<GetNextTestCase<UniqueDatasetParams>> GetNextTestCases() {
CreateTensors<int64>(TensorShape({1}), {})},
{/*dataset_params=*/StringParams(),
/*expected_outputs=*/
CreateTensors<string>(TensorShape({1}), {{"one"},
{"One"},
{"two"},
{"three"},
{"five"},
{"eight"},
{"thirteen"},
{"twenty-one"},
{"thirty-four"}})}};
CreateTensors<tstring>(TensorShape({1}), {{"one"},
{"One"},
{"two"},
{"three"},
{"five"},
{"eight"},
{"thirteen"},
{"twenty-one"},
{"thirty-four"}})}};
}
ITERATOR_GET_NEXT_TEST_P(UniqueDatasetOpTest, UniqueDatasetParams,

2
tensorflow/core/kernels/data/text_line_dataset_op_test.cc
View File

@ -66,7 +66,7 @@ class TextLineDatasetParams : public DatasetParams {
class TextLineDatasetOpTest : public DatasetOpsTestBase {};
Status CreateTestFiles(const std::vector<tstring>& filenames,
const std::vector<string>& contents,
const std::vector<tstring>& contents,
CompressionType compression_type) {
if (filenames.size() != contents.size()) {
return tensorflow::errors::InvalidArgument(

4
tensorflow/core/kernels/decode_proto_op.cc
View File

@ -810,7 +810,7 @@ class DecodeProtoOp : public OpKernel {
private:
// Copy a serialized message to binary, e.g. to handle text proto inputs.
void ReserializeMessage(OpKernelContext* ctx, const string& buf,
void ReserializeMessage(OpKernelContext* ctx, const tstring& buf,
tstring* binary_buf) {
// Handle text protos by translating them to binary.
std::unique_ptr<Message> message(message_prototype_->New());
@ -831,7 +831,7 @@ class DecodeProtoOp : public OpKernel {
}
// Count the number of occurrences of each requested field in a message batch.
void CountFields(OpKernelContext* ctx, int message_index, const string& buf,
void CountFields(OpKernelContext* ctx, int message_index, const tstring& buf,
Tensor* sizes_tensor, std::vector<int32>* max_sizes) {
int field_count = fields_.size();

6
tensorflow/core/kernels/deserialize_sparse_string_op.cc
View File

@ -210,7 +210,7 @@ class DeserializeSparseOp : public OpKernel {
}
private:
Status Deserialize(const string& serialized, Tensor* result) {
Status Deserialize(const tstring& serialized, Tensor* result) {
TensorProto proto;
if (!ParseProtoUnlimited(&proto, serialized)) {
return errors::InvalidArgument("Could not parse serialized proto");
@ -224,8 +224,8 @@ class DeserializeSparseOp : public OpKernel {
}
Status GetAndValidateSparseTensor(
const string& serialized_indices, const string& serialized_values,
const string& serialized_shape, DataType values_dtype, int index,
const tstring& serialized_indices, const tstring& serialized_values,
const tstring& serialized_shape, DataType values_dtype, int index,
Tensor* output_indices, Tensor* output_values, Tensor* output_shape) {
// Deserialize and validate the indices.
TF_RETURN_IF_ERROR(this->Deserialize(serialized_indices, output_indices));

30
tensorflow/core/kernels/example_parsing_ops.cc
View File

@ -230,7 +230,7 @@ class ParseExampleOp : public OpKernel {
Status ParseExampleScalar(const example::FastParseExampleConfig& config,
const Tensor* serialized, OpKernelContext* ctx,
example::Result* result) const {
const string& serialized_proto = serialized->scalar<tstring>()();
const tstring& serialized_proto = serialized->scalar<tstring>()();
return FastParseSingleExample(config, serialized_proto, result);
}
@ -357,7 +357,7 @@ class ParseSingleExampleOp : public OpKernel {
config.sparse.push_back({attrs_.sparse_keys[d], attrs_.sparse_types[d]});
}
const string& serialized_proto = serialized->scalar<tstring>()();
const tstring& serialized_proto = serialized->scalar<tstring>()();
OP_REQUIRES_OK(ctx,
FastParseSingleExample(config, serialized_proto, &result));
@ -567,8 +567,8 @@ class ParseSequenceExampleOp : public OpKernel {
const OpInputList& context_dense_defaults) const {
example::FastParseExampleConfig config;
for (int d = 0; d < attrs_.num_context_dense; ++d) {
const string& key = dense_keys ? dense_keys->flat<tstring>()(d)
: attrs_.context_dense_keys[d];
const tstring& key = dense_keys ? dense_keys->flat<tstring>()(d)
: attrs_.context_dense_keys[d];
config.dense.push_back({key, attrs_.context_dense_types[d],
attrs_.context_dense_shapes[d],
context_dense_defaults[d],
@ -576,8 +576,8 @@ class ParseSequenceExampleOp : public OpKernel {
0 /*attrs_.context_elements_per_stride[d] */});
}
for (int d = 0; d < attrs_.num_context_sparse; ++d) {
const string& key = sparse_keys ? sparse_keys->flat<tstring>()(d)
: attrs_.context_sparse_keys[d];
const tstring& key = sparse_keys ? sparse_keys->flat<tstring>()(d)
: attrs_.context_sparse_keys[d];
config.sparse.push_back({key, attrs_.context_sparse_types[d]});
}
for (int d = 0; d < attrs_.num_context_ragged; ++d) {
@ -594,8 +594,8 @@ class ParseSequenceExampleOp : public OpKernel {
const Tensor* feature_list_dense_missing_assumed_empty) const {
example::FastParseExampleConfig config;
for (int d = 0; d < attrs_.num_feature_list_dense; ++d) {
const string& key = dense_keys ? dense_keys->flat<tstring>()(d)
: attrs_.feature_list_dense_keys[d];
const tstring& key = dense_keys ? dense_keys->flat<tstring>()(d)
: attrs_.feature_list_dense_keys[d];
bool missing_assumed_empty =
feature_list_dense_missing_assumed_empty
? feature_list_dense_missing_assumed_empty->flat<bool>()(d)
@ -608,8 +608,8 @@ class ParseSequenceExampleOp : public OpKernel {
0 /*attrs_.feature_list_elements_per_stride[d] */});
}
for (int d = 0; d < attrs_.num_feature_list_sparse; ++d) {
const string& key = sparse_keys ? sparse_keys->flat<tstring>()(d)
: attrs_.feature_list_sparse_keys[d];
const tstring& key = sparse_keys ? sparse_keys->flat<tstring>()(d)
: attrs_.feature_list_sparse_keys[d];
config.sparse.push_back({key, attrs_.feature_list_sparse_types[d]});
}
for (int d = 0; d < attrs_.num_feature_list_ragged; ++d) {
@ -909,7 +909,7 @@ class ParseSingleSequenceExampleOp : public OpKernel {
errors::InvalidArgument("Could not parse example input, value: '",
serialized_t(), "'"));
const string& name = (has_debug_name) ? debug_name_t() : "<unknown>";
const tstring& name = (has_debug_name) ? debug_name_t() : "<unknown>";
const Features& context = ex.context();
const auto& context_dict = context.feature();
@ -925,7 +925,7 @@ class ParseSingleSequenceExampleOp : public OpKernel {
}
for (int d = 0; d < attrs_.num_context_dense; ++d) {
const string& key = context_dense_keys_t[d];
const tstring& key = context_dense_keys_t[d];
const DataType& dtype = attrs_.context_dense_types[d];
const TensorShape& shape = attrs_.context_dense_shapes[d];
@ -955,7 +955,7 @@ class ParseSingleSequenceExampleOp : public OpKernel {
// Context Sparse ----------------------------------------------------------
for (int d = 0; d < attrs_.num_context_sparse; ++d) {
const string& key = context_sparse_keys_t[d];
const tstring& key = context_sparse_keys_t[d];
const DataType& dtype = attrs_.context_sparse_types[d];
const auto& feature_found = context_dict.find(key);
@ -1014,7 +1014,7 @@ class ParseSingleSequenceExampleOp : public OpKernel {
FeatureList empty_feature_list; // Placeholder for missing FLs
for (int d = 0; d < attrs_.num_feature_list_dense; ++d) {
const string& key = feature_list_dense_keys_t[d];
const tstring& key = feature_list_dense_keys_t[d];
const DataType& dtype = attrs_.feature_list_dense_types[d];
const TensorShape& shape = attrs_.feature_list_dense_shapes[d];
@ -1061,7 +1061,7 @@ class ParseSingleSequenceExampleOp : public OpKernel {
// Feature List Sparse -----------------------------------------------------
for (int d = 0; d < attrs_.num_feature_list_sparse; ++d) {
const string& key = feature_list_sparse_keys_t[d];
const tstring& key = feature_list_sparse_keys_t[d];
const DataType& dtype = attrs_.feature_list_sparse_types[d];
const auto& feature_list_found = feature_list_dict.find(key);

2
tensorflow/core/kernels/example_parsing_ops_test.cc
View File

@ -223,7 +223,7 @@ static Graph* ParseExampleV2(int batch_size, int num_keys, int feature_size) {
std::vector<DataType> ragged_split_types;
std::vector<PartialTensorShape> dense_shapes;
Tensor keys_t(DT_STRING, {static_cast<int32>(num_keys)});
auto keys_flat = keys_t.flat<string>();
auto keys_flat = keys_t.flat<tstring>();
Options opt;
for (int i = 0; i < num_keys; ++i) {
keys_flat(i) = strings::Printf("feature_%d", i);

4
tensorflow/core/kernels/ragged_tensor_to_tensor_op.cc
View File

@ -406,12 +406,12 @@ void copy_array(VALUE_TYPE* dst, const VALUE_TYPE* src, INDEX_TYPE size) {
}
template <>
void copy_array<string, int64>(string* dst, const string* src, int64 size) {
void copy_array<tstring, int64>(tstring* dst, const tstring* src, int64 size) {
slow_copy_array(dst, src, size);
}
template <>
void copy_array<string, int32>(string* dst, const string* src, int32 size) {
void copy_array<tstring, int32>(tstring* dst, const tstring* src, int32 size) {
slow_copy_array(dst, src, size);
}

2
tensorflow/core/kernels/scatter_functor.h
View File

@ -362,7 +362,7 @@ struct ScatterFunctorBase<CPUDevice, T, Index, scatter_op::UpdateOp::ASSIGN> {
// indices and params sizes were validated in DoCompute().
const Index N = static_cast<Index>(indices.size());
const Index limit = static_cast<Index>(params.dimension(0));
if (!std::is_same<T, string>::value) {
if (!std::is_same<T, tstring>::value) {
for (Index i = 0; i < N; i++) {
// Grab the index and check its validity. Do this carefully,
// to avoid checking the value and grabbing it again from

6
tensorflow/core/kernels/strided_slice_op.h
View File

@ -68,10 +68,10 @@ struct InitOutput<ResourceHandle, NDIMS, Device> {
};
template <int NDIMS, typename Device>
struct InitOutput<string, NDIMS, Device> {
struct InitOutput<tstring, NDIMS, Device> {
static void run(const Device& d,
typename TTypes<string, NDIMS>::Tensor output) {
output.device(d) = output.constant(string());
typename TTypes<tstring, NDIMS>::Tensor output) {
output.device(d) = output.constant(tstring());
}
};

4
tensorflow/core/kernels/tensor_cord_test.cc
View File

@ -205,7 +205,7 @@ TEST(TensorCordTest, MoveConstructor) {
void TensorCopyFromTensorBenchmark(benchmark::State& state, int num_elem,
int string_size) {
Tensor strings(DT_STRING, {num_elem});
auto t = strings.flat<string>();
auto t = strings.flat<tstring>();
for (int i = 0; i < num_elem; ++i) {
t(i).insert(0, string_size, 'a');
}
@ -217,7 +217,7 @@ void TensorCopyFromTensorBenchmark(benchmark::State& state, int num_elem,
void TensorCordFromTensorBenchmark(benchmark::State& state, int num_elem,
int string_size) {
Tensor strings(DT_STRING, {num_elem});
auto t = strings.flat<string>();
auto t = strings.flat<tstring>();
for (int i = 0; i < num_elem; ++i) {
t(i).insert(0, string_size, 'a');
}

4
tensorflow/core/util/batch_util.cc
View File

@ -105,8 +105,8 @@ void HandleSliceToElement(Tensor* parent, Tensor* element, int64 index,
}
template <>
void HandleSliceToElement<string>(Tensor* parent, Tensor* element, int64 index,
bool can_move) {
void HandleSliceToElement<tstring>(Tensor* parent, Tensor* element, int64 index,
bool can_move) {
auto parent_as_matrix = parent->flat_outer_dims<tstring>();
auto element_flat = element->flat<tstring>();
if (can_move) {

2
tensorflow/core/util/example_proto_fast_parsing.cc
View File

@ -1078,7 +1078,7 @@ void CopySparseBufferToTensor(DataType dtype, size_t offset, SparseBuffer* src,
}
case DT_STRING: {
std::move(src->bytes_list.begin(), src->bytes_list.end(),
dst->flat<string>().data() + offset);
dst->flat<tstring>().data() + offset);
break;
}
default:

12
tensorflow/core/util/example_proto_helper.h
View File

@ -226,9 +226,9 @@ struct ParseSingleExampleAttrs {
return FinishInit();
}
std::vector<string> sparse_keys;
std::vector<tstring> sparse_keys;
std::vector<DataType> sparse_types;
std::vector<string> dense_keys;
std::vector<tstring> dense_keys;
std::vector<DataType> dense_types;
std::vector<PartialTensorShape> dense_shapes;
std::vector<bool> variable_length;
@ -302,10 +302,10 @@ struct ParseSequenceExampleAttrs {
int64 num_feature_list_sparse;
int64 num_feature_list_dense;
int64 num_feature_list_ragged;
std::vector<string> context_sparse_keys;
std::vector<string> context_dense_keys;
std::vector<string> feature_list_sparse_keys;
std::vector<string> feature_list_dense_keys;
std::vector<tstring> context_sparse_keys;
std::vector<tstring> context_dense_keys;
std::vector<tstring> feature_list_sparse_keys;
std::vector<tstring> feature_list_dense_keys;
std::vector<DataType> context_sparse_types;
std::vector<DataType> context_dense_types;
std::vector<TensorShape> context_dense_shapes;

16
tensorflow/core/util/proto/decode.h
View File

@ -381,13 +381,15 @@ inline Status ReadGroupBytes(CodedInputStream* input, int field_number,
#else // USE_TSTRING
StringOutputStream string_stream(data);
#endif // USE_TSTRING
CodedOutputStream out(&string_stream);
if (!WireFormatLite::SkipField(
input,
WireFormatLite::MakeTag(field_number,
WireFormatLite::WIRETYPE_START_GROUP),
&out)) {
return errors::DataLoss("Failed reading group");
{
CodedOutputStream out(&string_stream);
if (!WireFormatLite::SkipField(
input,
WireFormatLite::MakeTag(field_number,
WireFormatLite::WIRETYPE_START_GROUP),
&out)) {
return errors::DataLoss("Failed reading group");
}
}
#ifdef USE_TSTRING
*data = buf;

6
tensorflow/python/lib/core/py_seq_tensor.cc
View File

@ -482,7 +482,7 @@ typedef Converter<Eigen::half> NumpyHalfConverter;
// String support
template <>
struct ConverterTraits<string> {
struct ConverterTraits<tstring> {
static const tensorflow::DataType kTypeEnum = DT_STRING;
static const char* ConvertScalar(PyObject* v, tstring* out) {
@ -509,7 +509,7 @@ struct ConverterTraits<string> {
}
};
typedef Converter<string> StringConverter;
typedef Converter<tstring> StringConverter;
// Converts Python object `c` that should hold a Python string into a
// C++ string in *out. Returns nullptr on success, or a message on error.
@ -521,7 +521,7 @@ tstring PyRepr(PyObject* obj) {
Safe_PyObjectPtr repr_obj = make_safe(PyObject_Repr(obj));
if (repr_obj) {
tstring repr_str;
if (ConverterTraits<string>::ConvertScalar(repr_obj.get(), &repr_str) ==
if (ConverterTraits<tstring>::ConvertScalar(repr_obj.get(), &repr_str) ==
nullptr) {
return repr_str;
}