Removed USE_TSTRING usage.

With tensorflow::tstring enabled, these are now superfluous.  Also updated Tensor::scalar<std::string>() to yield a build error instead of a runtime error.

PiperOrigin-RevId: 292079208
Change-Id: I599caed1f1866f366f0c378c9cc680c77eaba47f
This commit is contained in:
Dero Gharibian 2020-01-28 22:23:15 -08:00 committed by TensorFlower Gardener
parent 9d41ea557a
commit 38cdb9ff85
31 changed files with 7 additions and 143 deletions

View File

@ -114,7 +114,6 @@ bool GrpcMaybeParseProto(grpc::ByteBuffer* src, string* dst) {
return true;
}
#ifdef USE_TSTRING
// GrpcMaybeParseProto simply copies bytes into the tstring.
bool GrpcMaybeParseProto(grpc::ByteBuffer* src, tstring* dst) {
dst->clear();
@ -128,6 +127,5 @@ bool GrpcMaybeParseProto(grpc::ByteBuffer* src, tstring* dst) {
}
return true;
}
#endif // USE_TSTRING
} // namespace tensorflow

View File

@ -50,14 +50,12 @@ bool HasFeature<string>(const string& key, const Features& features) {
(it->second.kind_case() == Feature::KindCase::kBytesList);
}
#ifdef USE_TSTRING
template <>
bool HasFeature<tstring>(const string& key, const Features& features) {
auto it = features.feature().find(key);
return (it != features.feature().end()) &&
(it->second.kind_case() == Feature::KindCase::kBytesList);
}
#endif
bool HasFeatureList(const string& key,
const SequenceExample& sequence_example) {
@ -88,13 +86,11 @@ protobuf::RepeatedField<float>* GetFeatureValues<float>(Feature* feature) {
return feature->mutable_float_list()->mutable_value();
}
#ifdef USE_TSTRING
template <>
const protobuf::RepeatedPtrField<string>& GetFeatureValues<tstring>(
const Feature& feature) {
return feature.bytes_list().value();
}
#endif
template <>
const protobuf::RepeatedPtrField<string>& GetFeatureValues<string>(
@ -102,13 +98,11 @@ const protobuf::RepeatedPtrField<string>& GetFeatureValues<string>(
return feature.bytes_list().value();
}
#ifdef USE_TSTRING
template <>
protobuf::RepeatedPtrField<string>* GetFeatureValues<tstring>(
Feature* feature) {
return feature->mutable_bytes_list()->mutable_value();
}
#endif
template <>
protobuf::RepeatedPtrField<string>* GetFeatureValues<string>(Feature* feature) {
@ -142,12 +136,10 @@ void ClearFeatureValues<string>(Feature* feature) {
feature->mutable_bytes_list()->Clear();
}
#ifdef USE_TSTRING
template <>
void ClearFeatureValues<tstring>(Feature* feature) {
feature->mutable_bytes_list()->Clear();
}
#endif
template <>
Features* GetFeatures<Features>(Features* proto) {
@ -188,18 +180,14 @@ template <>
const protobuf::RepeatedPtrField<string>& GetFeatureValues<string>(
const Feature& feature);
#ifdef USE_TSTRING
template <>
const protobuf::RepeatedPtrField<string>& GetFeatureValues<tstring>(
const Feature& feature);
#endif
template <>
protobuf::RepeatedPtrField<string>* GetFeatureValues<string>(Feature* feature);
#ifdef USE_TSTRING
template <>
protobuf::RepeatedPtrField<string>* GetFeatureValues<tstring>(Feature* feature);
#endif
} // namespace tensorflow

View File

@ -149,12 +149,10 @@ struct RepeatedFieldTrait<float> {
using Type = protobuf::RepeatedField<float>;
};
#ifdef USE_TSTRING
template <>
struct RepeatedFieldTrait<tstring> {
using Type = protobuf::RepeatedPtrField<string>;
};
#endif
template <>
struct RepeatedFieldTrait<string> {
@ -193,10 +191,8 @@ struct is_string<string> : std::true_type {};
template <>
struct is_string<::tensorflow::StringPiece> : std::true_type {};
#ifdef USE_TSTRING
template <>
struct is_string<tstring> : std::true_type {};
#endif
template <typename ValueType>
struct FeatureTrait<

View File

@ -481,7 +481,6 @@ DEFINE_SET_ATTR_VALUE_LIST(const std::vector<bool>&, b)
DEFINE_SET_ATTR_VALUE_LIST(std::initializer_list<bool>, b)
DEFINE_SET_ATTR_VALUE_BOTH(DataType, type)
#ifdef USE_TSTRING
void SetAttrValue(const tstring& value, AttrValue* out) {
out->set_s(value.data(), value.size());
}
@ -492,7 +491,6 @@ void SetAttrValue(gtl::ArraySlice<tstring> value, AttrValue* out) {
out->mutable_list()->add_s(v.data(), v.size());
}
}
#endif
void SetAttrValue(StringPiece value, AttrValue* out) {
out->set_s(value.data(), value.size());

View File

@ -154,9 +154,6 @@ class GraphDefBuilderWrapper {
return Status::OK();
}
#ifdef USE_TSTRING
// TODO(dero): Temp guard to prevent duplicate declaration during tstring
// migration.
Status AddVector(const std::vector<string>& val, Node** output) {
Tensor val_t = Tensor(DataTypeToEnum<tstring>::v(),
TensorShape({static_cast<int64>(val.size())}));
@ -169,7 +166,6 @@ class GraphDefBuilderWrapper {
}
return Status::OK();
}
#endif // USE_TSTRING
// Adds a `Const` node for the given tensor value to the graph.
//

View File

@ -309,9 +309,7 @@ ATTR(const NameAttrList&)
ATTR(gtl::ArraySlice<StringPiece>)
ATTR(gtl::ArraySlice<const char*>)
ATTR(gtl::ArraySlice<string>)
#ifdef USE_TSTRING
ATTR(gtl::ArraySlice<tstring>)
#endif
ATTR(gtl::ArraySlice<int32>)
ATTR(gtl::ArraySlice<int64>)
ATTR(gtl::ArraySlice<float>)

View File

@ -111,9 +111,7 @@ class NodeDefBuilder {
NodeDefBuilder& Attr(StringPiece name, gtl::ArraySlice<StringPiece> value);
NodeDefBuilder& Attr(StringPiece name, gtl::ArraySlice<const char*> value);
NodeDefBuilder& Attr(StringPiece name, gtl::ArraySlice<string> value);
#ifdef USE_TSTRING
NodeDefBuilder& Attr(StringPiece name, gtl::ArraySlice<tstring> value);
#endif
NodeDefBuilder& Attr(StringPiece name, gtl::ArraySlice<int32> value);
NodeDefBuilder& Attr(StringPiece name, gtl::ArraySlice<int64> value);
NodeDefBuilder& Attr(StringPiece name, gtl::ArraySlice<float> value);

View File

@ -249,10 +249,8 @@ bool AttrSlice::EqualAttrs(AttrSlice other, Scratch* scratch) const {
} \
return true; \
}
#ifdef USE_TSTRING
DEFINE_GET_ATTR(tstring, s, "string", emplace_back, v, ;)
DEFINE_TRY_GET_ATTR(tstring, s, "string", emplace_back, v, ;)
#endif
DEFINE_GET_ATTR(string, s, "string", emplace_back, v, ;)
DEFINE_TRY_GET_ATTR(string, s, "string", emplace_back, v, ;)
DEFINE_GET_ATTR(int64, i, "int", emplace_back, v, ;)

View File

@ -84,7 +84,6 @@ struct NumTraits<tensorflow::bfloat16>
}
};
#ifdef USE_TSTRING
template <>
struct NumTraits<tensorflow::tstring> : GenericNumTraits<tensorflow::tstring> {
enum {
@ -104,7 +103,6 @@ struct NumTraits<tensorflow::tstring> : GenericNumTraits<tensorflow::tstring> {
static inline tensorflow::tstring infinity();
static inline tensorflow::tstring quiet_NaN();
};
#endif // USE_TSTRING
using ::tensorflow::operator==;
using ::tensorflow::operator!=;

View File

@ -228,16 +228,7 @@ void ReaderBase::SaveBaseState(ReaderBaseState* state) const {
state->set_work_started(work_started_);
state->set_work_finished(work_finished_);
state->set_num_records_produced(num_records_produced_);
// Unfortunately, external proto does not accept string_view.
#if defined(PLATFORM_GOOGLE)
// TODO(dero): Remove NOLINT after USE_TSTRING is enabled. The external proto
// compiler does not create an overloaded set method that accepts
// absl::string_view, and string_view to std::string is an explicit
// conversion.
state->set_current_work(StringPiece(work_)); // NOLINT
#else
state->set_current_work(string(work_));
#endif
state->set_current_work(work_.data(), work_.size());
}
tstring ReaderBase::KeyName(const tstring& key) const {

View File

@ -862,33 +862,22 @@ typename TTypes<T, NDIMS>::UnalignedConstTensor Tensor::unaligned_shaped(
template <typename T>
typename TTypes<T>::Scalar Tensor::scalar() {
static_assert(
!std::is_same<T, std::string>::value,
"std::string is no longer a scalar type, use tensorflow::tstring");
CheckIsAlignedAndSingleElement();
return typename TTypes<T>::Scalar(base<T>());
}
#ifdef USE_TSTRING
template <>
inline typename TTypes<std::string>::Scalar Tensor::scalar<std::string>() {
LOG(FATAL)
<< "std::string is no longer a scalar type, use tensorflow::tstring";
}
#endif // USE_TSTRING
template <typename T>
typename TTypes<T>::ConstScalar Tensor::scalar() const {
static_assert(
!std::is_same<T, std::string>::value,
"std::string is no longer a scalar type, use tensorflow::tstring");
CheckIsAlignedAndSingleElement();
return typename TTypes<T>::ConstScalar(base<T>());
}
#ifdef USE_TSTRING
template <>
inline typename TTypes<std::string>::ConstScalar Tensor::scalar<std::string>()
const {
LOG(FATAL)
<< "std::string is no longer a scalar type, use tensorflow::tstring";
}
#endif // USE_TSTRING
template <typename T, size_t NDIMS>
typename TTypes<T, NDIMS>::Tensor Tensor::flat_inner_dims() {
return shaped<T, NDIMS>(ComputeFlatInnerDims(shape_.dim_sizes(), NDIMS));

View File

@ -188,9 +188,7 @@ Status BufferedInputStream::ReadAll(T* result) {
}
template Status BufferedInputStream::ReadAll<string>(string* result);
#ifdef USE_TSTRING
template Status BufferedInputStream::ReadAll<tstring>(tstring* result);
#endif // USE_TSTRING
Status BufferedInputStream::Reset() {
TF_RETURN_IF_ERROR(input_stream_->Reset());

View File

@ -106,10 +106,8 @@ class BufferedInputStream : public InputStreamInterface {
#ifndef SWIG
extern template tensorflow::Status BufferedInputStream::ReadAll<string>(
string* result);
#ifdef USE_TSTRING
extern template tensorflow::Status BufferedInputStream::ReadAll<tstring>(
tstring* result);
#endif // USE_TSTRING
#endif // SWIG
} // namespace io

View File

@ -73,9 +73,7 @@ Status InputBuffer::ReadLine(T* result) {
}
template Status InputBuffer::ReadLine<string>(string* result);
#ifdef USE_TSTRING
template Status InputBuffer::ReadLine<tstring>(tstring* result);
#endif // USE_TSTRING
Status InputBuffer::ReadNBytes(int64 bytes_to_read, string* result) {
result->clear();

View File

@ -111,9 +111,7 @@ class InputBuffer {
// Explicit instantiations defined in inputbuffer.cc.
extern template Status InputBuffer::ReadLine<string>(string* result);
#ifdef USE_TSTRING
extern template Status InputBuffer::ReadLine<tstring>(tstring* result);
#endif // USE_TSTRING
// Inlined for performance.
inline Status InputBuffer::ReadVarint32(uint32* result) {

View File

@ -424,12 +424,10 @@ template bool WriteImageToBuffer<string>(
const void* image, int width, int height, int row_bytes, int num_channels,
int channel_bits, int compression, string* png_string,
const std::vector<std::pair<string, string> >* metadata);
#ifdef USE_TSTRING
template bool WriteImageToBuffer<tstring>(
const void* image, int width, int height, int row_bytes, int num_channels,
int channel_bits, int compression, tstring* png_string,
const std::vector<std::pair<string, string> >* metadata);
#endif // USE_TSTRING
} // namespace png
} // namespace tensorflow

View File

@ -105,12 +105,10 @@ extern template bool WriteImageToBuffer<string>(
const void* image, int width, int height, int row_bytes, int num_channels,
int channel_bits, int compression, string* png_string,
const std::vector<std::pair<string, string> >* metadata);
#ifdef USE_TSTRING
extern template bool WriteImageToBuffer<tstring>(
const void* image, int width, int height, int row_bytes, int num_channels,
int channel_bits, int compression, tstring* png_string,
const std::vector<std::pair<string, string> >* metadata);
#endif // USE_TSTRING
} // namespace png
} // namespace tensorflow

View File

@ -214,13 +214,11 @@ template Status EncodeAudioAsS16LEWav<string>(const float* audio,
size_t num_channels,
size_t num_frames,
string* wav_string);
#ifdef USE_TSTRING
template Status EncodeAudioAsS16LEWav<tstring>(const float* audio,
size_t sample_rate,
size_t num_channels,
size_t num_frames,
tstring* wav_string);
#endif // USE_TSTRING
Status DecodeLin16WaveAsFloatVector(const string& wav_string,
std::vector<float>* float_values,

View File

@ -52,13 +52,11 @@ extern template Status EncodeAudioAsS16LEWav<string>(const float* audio,
size_t num_channels,
size_t num_frames,
string* wav_string);
#ifdef USE_TSTRING
extern template Status EncodeAudioAsS16LEWav<tstring>(const float* audio,
size_t sample_rate,
size_t num_channels,
size_t num_frames,
tstring* wav_string);
#endif // USE_TSTRING
// Decodes the little-endian signed 16-bit PCM WAV file data (aka LIN16
// encoding) into a float Tensor. The channels are encoded as the lowest

View File

@ -199,11 +199,9 @@ template Status Base64Encode<string>(StringPiece source, string* encoded);
template Status Base64Encode<string>(StringPiece source, bool with_padding,
string* encoded);
#ifdef USE_TSTRING
template Status Base64Decode<tstring>(StringPiece data, tstring* decoded);
template Status Base64Encode<tstring>(StringPiece source, tstring* encoded);
template Status Base64Encode<tstring>(StringPiece source, bool with_padding,
tstring* encoded);
#endif // USE_TSTRING
} // namespace tensorflow

View File

@ -43,7 +43,6 @@ extern template Status Base64Encode<string>(StringPiece source,
extern template Status Base64Encode<string>(StringPiece source,
bool with_padding, string* encoded);
#ifdef USE_TSTRING
extern template Status Base64Decode<tstring>(StringPiece data,
tstring* decoded);
extern template Status Base64Encode<tstring>(StringPiece source,
@ -51,7 +50,6 @@ extern template Status Base64Encode<tstring>(StringPiece source,
extern template Status Base64Encode<tstring>(StringPiece source,
bool with_padding,
tstring* encoded);
#endif // USE_TSTRING
} // namespace tensorflow

View File

@ -107,13 +107,11 @@ void PutVarint32(string* dst, uint32 v) {
dst->append(buf, ptr - buf);
}
#ifdef USE_TSTRING
void PutVarint32(tstring* dst, uint32 v) {
char buf[5];
char* ptr = EncodeVarint32(buf, v);
dst->append(buf, ptr - buf);
}
#endif
char* EncodeVarint64(char* dst, uint64 v) {
static const int B = 128;
@ -132,13 +130,11 @@ void PutVarint64(string* dst, uint64 v) {
dst->append(buf, ptr - buf);
}
#ifdef USE_TSTRING
void PutVarint64(tstring* dst, uint64 v) {
char buf[10];
char* ptr = EncodeVarint64(buf, v);
dst->append(buf, ptr - buf);
}
#endif
int VarintLength(uint64_t v) {
int len = 1;

View File

@ -42,11 +42,9 @@ inline uint64 Hash64(const string& str) {
return Hash64(str.data(), str.size());
}
#ifdef USE_TSTRING
inline uint64 Hash64(const tstring& str) {
return Hash64(str.data(), str.size());
}
#endif // USE_TSTRING
inline uint64 Hash64Combine(uint64 a, uint64 b) {
return a ^ (b + 0x9e3779b97f4a7800ULL + (a << 10) + (a >> 4));
@ -101,14 +99,12 @@ struct hash<string> {
}
};
#ifdef USE_TSTRING
template <>
struct hash<tstring> {
size_t operator()(const tstring& s) const {
return static_cast<size_t>(Hash64(s.data(), s.size()));
}
};
#endif // USE_TSTRING
template <>
struct hash<StringPiece> {
@ -127,7 +123,6 @@ struct hash<std::pair<T, U>> {
} // namespace tensorflow
#ifdef USE_TSTRING
namespace std {
template <>
struct hash<tensorflow::tstring> {
@ -136,6 +131,5 @@ struct hash<tensorflow::tstring> {
}
};
} // namespace std
#endif // USE_TSTRING
#endif // TENSORFLOW_CORE_PLATFORM_HASH_H_

View File

@ -20,7 +20,6 @@ namespace tensorflow {
const char* kProtobufInt64Typename = "::tensorflow::protobuf_int64";
const char* kProtobufUint64Typename = "::tensorflow::protobuf_uint64";
#ifdef USE_TSTRING
TStringOutputStream::TStringOutputStream(tstring* target) : target_(target) {}
bool TStringOutputStream::Next(void** data, int* size) {
@ -55,6 +54,5 @@ void TStringOutputStream::BackUp(int count) {
}
int64_t TStringOutputStream::ByteCount() const { return target_->size(); }
#endif // USE_TSTRING
} // namespace tensorflow

View File

@ -58,12 +58,10 @@ bool ParseProtoUnlimited(protobuf::MessageLite* proto,
const string& serialized);
bool ParseProtoUnlimited(protobuf::MessageLite* proto, const void* serialized,
size_t size);
#ifdef USE_TSTRING
inline bool ParseProtoUnlimited(protobuf::MessageLite* proto,
const tstring& serialized) {
return ParseProtoUnlimited(proto, serialized.data(), serialized.size());
}
#endif // USE_TSTRING
// Returns the string value for the value of a string or bytes protobuf field.
inline const string& ProtobufStringToString(const string& s) { return s; }
@ -87,16 +85,11 @@ inline void SetProtobufStringSwapAllowed(string* src, Cord* dest) {
inline bool SerializeToTString(const protobuf::MessageLite& proto,
tstring* output) {
#ifdef USE_TSTRING
size_t size = proto.ByteSizeLong();
output->resize_uninitialized(size);
return proto.SerializeToArray(output->data(), static_cast<int>(size));
#else // USE_TSTRING
return proto.SerializeToString(output);
#endif // USE_TSTRING
}
#ifdef USE_TSTRING
// Analogue to StringOutputStream for tstring.
class TStringOutputStream : public protobuf::io::ZeroCopyOutputStream {
public:
@ -115,9 +108,6 @@ class TStringOutputStream : public protobuf::io::ZeroCopyOutputStream {
tstring* target_;
};
#else // USE_TSTRING
typedef protobuf::io::StringOutputStream TStringOutputStream;
#endif // USE_TSTRING
} // namespace tensorflow

View File

@ -124,12 +124,8 @@ class AlphaNum {
AlphaNum(const StringPiece &pc) : piece_(pc) {} // NOLINT(runtime/explicit)
AlphaNum(const tensorflow::string &str) // NOLINT(runtime/explicit)
: piece_(str) {}
#ifdef USE_TSTRING
// TODO(dero): Temp guard to prevent duplicate declaration during tstring
// migration.
AlphaNum(const tensorflow::tstring &str) // NOLINT(runtime/explicit)
: piece_(str) {}
#endif
template <typename A>
AlphaNum(const std::basic_string<char, std::char_traits<char>, A> &str)
: piece_(str) {} // NOLINT(runtime/explicit)

View File

@ -177,7 +177,6 @@ bool DecodeStringList(const Cord& src, string* strings, int64 n) {
return true;
}
#ifdef USE_TSTRING
bool DecodeStringList(const Cord& src, tstring* strings, int64 n) {
std::vector<uint32> sizes(n);
CordReader reader(src);
@ -200,7 +199,6 @@ bool DecodeStringList(const Cord& src, tstring* strings, int64 n) {
}
return true;
}
#endif // USE_TSTRING
void CopyFromArray(Cord* c, const char* base, size_t bytes) {
c->CopyFrom(base, bytes);

View File

@ -24,10 +24,6 @@ limitations under the License.
#include "tensorflow/core/platform/cord.h"
#include "tensorflow/core/platform/ctstring.h"
#define USE_TSTRING
#ifdef USE_TSTRING
// TODO(dero): This include is temporary, and will be superfluous once
// absl::string_view is aliased to std::string_view.
#include "absl/strings/string_view.h"
@ -597,14 +593,4 @@ inline std::ostream& operator<<(std::ostream& o, const tstring& str) {
} // namespace tensorflow
#else // USE_TSTRING
namespace tensorflow {
typedef std::string tstring;
} // namespace tensorflow
#endif // USE_TSTRING
#endif // TENSORFLOW_CORE_PLATFORM_TSTRING_H_

View File

@ -1839,16 +1839,10 @@ inline int ParseBytesFeature(protobuf::io::CodedInputStream* stream,
if (out == nullptr) {
stream->Skip(bytes_length);
} else {
#ifdef USE_TSTRING
out->resize_uninitialized(bytes_length);
if (!stream->ReadRaw(out->data(), bytes_length)) {
return -1;
}
#else // USE_TSTRING
if (!stream->ReadString(out, bytes_length)) {
return -1;
}
#endif // USE_TSTRING
out++;
}
num_elements++;

View File

@ -339,7 +339,6 @@ inline Status ReadPrimitive(CodedInputStream* input, int index, void* data) {
inline Status ReadBytes(CodedInputStream* input, int index, void* datap) {
tstring* data = reinterpret_cast<tstring*>(datap) + index;
#ifdef USE_TSTRING
uint32 length;
if (!input->ReadVarint32(&length)) {
return errors::DataLoss("Failed reading bytes");
@ -350,11 +349,6 @@ inline Status ReadBytes(CodedInputStream* input, int index, void* datap) {
if (!input->ReadRaw(data->data(), length)) {
return errors::DataLoss("Failed reading bytes");
}
#else // USE_TSTRING
if (!WireFormatLite::ReadBytes(input, data)) {
return errors::DataLoss("Failed reading bytes");
}
#endif // USE_TSTRING
return Status::OK();
}
@ -369,7 +363,6 @@ inline Status ReadGroupBytes(CodedInputStream* input, int field_number,
// on input->IsFlat() == true and using input->GetDirectBufferPointer()
// with input->CurrentPosition().
tstring* data = reinterpret_cast<tstring*>(datap) + index;
#ifdef USE_TSTRING
// TODO(dero): To mitigate the string to tstring copy, we can implement our
// own scanner as described above. We would first need to obtain the length
// in an initial pass and resize/reserve the tstring. But, given that
@ -378,9 +371,6 @@ inline Status ReadGroupBytes(CodedInputStream* input, int field_number,
// TYPE_GROUP tag, we use std::string as a read buffer.
string buf;
StringOutputStream string_stream(&buf);
#else // USE_TSTRING
StringOutputStream string_stream(data);
#endif // USE_TSTRING
{
CodedOutputStream out(&string_stream);
if (!WireFormatLite::SkipField(
@ -391,9 +381,7 @@ inline Status ReadGroupBytes(CodedInputStream* input, int field_number,
return errors::DataLoss("Failed reading group");
}
}
#ifdef USE_TSTRING
*data = buf;
#endif // USE_TSTRING
return Status::OK();
}

View File

@ -734,12 +734,8 @@ TEST(TensorBundleTest, StringTensors) {
// Requires a 64-bit length.
tstring* backing_string = long_string_tensor.flat<tstring>().data();
#ifdef USE_TSTRING
backing_string->resize_uninitialized(kLongLength);
std::char_traits<char>::assign(backing_string->data(), kLongLength, 'd');
#else // USE_TSTRING
backing_string->assign(kLongLength, 'd');
#endif // USE_TSTRING
TF_EXPECT_OK(writer.Add("long_scalar", long_string_tensor));
// Mixes in some floats.