experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[tf.data] Fix "passing result of std::move() as const reference argument" warning.

Browse Source 
To avoid an implicit string-to-tstring conversion that was causing the warning, this change switches to using tstring as the string type for reading a line from a text file, and adds the necessary tstring overload in `BufferedInputStream`.

PiperOrigin-RevId: 302128176
Change-Id: I55592893c61a66922285896f6c1a42a6ca8e4785
This commit is contained in:
Derek Murray 2020-03-20 17:03:05 -07:00 committed by TensorFlower Gardener
parent c984ec0b36
commit 7de80adca6
3 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tensorflow/core/kernels/data/text_line_dataset_op.cc
View File

@ -100,18 +100,17 @@ class TextLineDatasetOp::Dataset : public DatasetBase {
do { do {
// We are currently processing a file, so try to read the next line. // We are currently processing a file, so try to read the next line.
if (buffered_input_stream_) { if (buffered_input_stream_) {
string line_contents; Tensor line_contents(tstring{});
Status s = buffered_input_stream_->ReadLine(&line_contents); tstring& line_contents_str = line_contents.scalar<tstring>()();
Status s = buffered_input_stream_->ReadLine(&line_contents_str);
if (s.ok()) { if (s.ok()) {
// Produce the line as output. // Produce the line as output.
static monitoring::CounterCell* bytes_counter = static monitoring::CounterCell* bytes_counter =
metrics::GetTFDataBytesReadCounter( metrics::GetTFDataBytesReadCounter(
name_utils::OpName(TextLineDatasetOp::kDatasetType)); name_utils::OpName(TextLineDatasetOp::kDatasetType));
bytes_counter->IncrementBy(line_contents.size()); bytes_counter->IncrementBy(line_contents_str.size());
out_tensors->emplace_back(ctx->allocator({}), DT_STRING, out_tensors->push_back(std::move(line_contents));
TensorShape({}));
out_tensors->back().scalar<tstring>()() = line_contents;
*end_of_sequence = false; *end_of_sequence = false;
return Status::OK(); return Status::OK();
} else if (!errors::IsOutOfRange(s)) { } else if (!errors::IsOutOfRange(s)) {

20
tensorflow/core/lib/io/buffered_inputstream.cc
View File

@ -21,17 +21,17 @@ namespace tensorflow {
namespace io { namespace io {
BufferedInputStream::BufferedInputStream(InputStreamInterface* input_stream, BufferedInputStream::BufferedInputStream(InputStreamInterface* input_stream,
size_t buffer_size, size_t buffer_bytes,
bool owns_input_stream) bool owns_input_stream)
: input_stream_(input_stream), : input_stream_(input_stream),
size_(buffer_size), size_(buffer_bytes),
owns_input_stream_(owns_input_stream) { owns_input_stream_(owns_input_stream) {
buf_.reserve(size_); buf_.reserve(size_);
} }
BufferedInputStream::BufferedInputStream(RandomAccessFile* file, BufferedInputStream::BufferedInputStream(RandomAccessFile* file,
size_t buffer_size) size_t buffer_bytes)
: BufferedInputStream(new RandomAccessInputStream(file), buffer_size, : BufferedInputStream(new RandomAccessInputStream(file), buffer_bytes,
true) {} true) {}
BufferedInputStream::~BufferedInputStream() { BufferedInputStream::~BufferedInputStream() {
@ -56,7 +56,9 @@ Status BufferedInputStream::FillBuffer() {
return s; return s;
} }
Status BufferedInputStream::ReadLineHelper(string* result, bool include_eol) { template <typename StringType>
Status BufferedInputStream::ReadLineHelper(StringType* result,
bool include_eol) {
result->clear(); result->clear();
Status s; Status s;
while (true) { while (true) {
@ -70,13 +72,13 @@ Status BufferedInputStream::ReadLineHelper(string* result, bool include_eol) {
char c = buf_[pos_++]; char c = buf_[pos_++];
if (c == '\n') { if (c == '\n') {
if (include_eol) { if (include_eol) {
*result += c; result->append(1, c);
} }
return Status::OK(); return Status::OK();
} }
// We don't append '\r' to *result // We don't append '\r' to *result
if (c != '\r') { if (c != '\r') {
*result += c; result->append(1, c);
} }
} }
if (errors::IsOutOfRange(s) && !result->empty()) { if (errors::IsOutOfRange(s) && !result->empty()) {
@ -202,6 +204,10 @@ Status BufferedInputStream::ReadLine(string* result) {
return ReadLineHelper(result, false); return ReadLineHelper(result, false);
} }
Status BufferedInputStream::ReadLine(tstring* result) {
return ReadLineHelper(result, false);
}
string BufferedInputStream::ReadLineAsString() { string BufferedInputStream::ReadLineAsString() {
string result; string result;
ReadLineHelper(&result, true).IgnoreError(); ReadLineHelper(&result, true).IgnoreError();

4
tensorflow/core/lib/io/buffered_inputstream.h
View File

@ -66,6 +66,7 @@ class BufferedInputStream : public InputStreamInterface {
// file, we return an OUT_OF_RANGE error. Otherwise, we return // file, we return an OUT_OF_RANGE error. Otherwise, we return
// some other non-OK status. // some other non-OK status.
tensorflow::Status ReadLine(string* result); tensorflow::Status ReadLine(string* result);
tensorflow::Status ReadLine(tstring* result);
// Returns one text line of data until end-of-file or a '\n' is read. The '\n' // Returns one text line of data until end-of-file or a '\n' is read. The '\n'
// is included in the result. // is included in the result.
@ -86,7 +87,8 @@ class BufferedInputStream : public InputStreamInterface {
private: private:
tensorflow::Status FillBuffer(); tensorflow::Status FillBuffer();
tensorflow::Status ReadLineHelper(string* result, bool include_eol); template <typename StringType>
tensorflow::Status ReadLineHelper(StringType* result, bool include_eol);
InputStreamInterface* input_stream_; // not owned. InputStreamInterface* input_stream_; // not owned.
size_t size_; // buffer size. size_t size_; // buffer size.