experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[tf.data] Fix "passing result of std::move() as const reference argument" warning.

Browse Source 
To avoid an implicit string-to-tstring conversion that was causing the warning, this change switches to using tstring as the string type for reading a line from a text file, and adds the necessary tstring overload in `BufferedInputStream`.

PiperOrigin-RevId: 302128176
Change-Id: I55592893c61a66922285896f6c1a42a6ca8e4785
This commit is contained in:
Derek Murray 2020-03-20 17:03:05 -07:00 committed by TensorFlower Gardener
parent c984ec0b36
commit 7de80adca6
3 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tensorflow/core/kernels/data/text_line_dataset_op.cc
View File

@ -100,18 +100,17 @@ class TextLineDatasetOp::Dataset : public DatasetBase {
do {
// We are currently processing a file, so try to read the next line.
if (buffered_input_stream_) {
string line_contents;
Status s = buffered_input_stream_->ReadLine(&line_contents);
Tensor line_contents(tstring{});
tstring& line_contents_str = line_contents.scalar<tstring>()();
Status s = buffered_input_stream_->ReadLine(&line_contents_str);
if (s.ok()) {
// Produce the line as output.
static monitoring::CounterCell* bytes_counter =
metrics::GetTFDataBytesReadCounter(
name_utils::OpName(TextLineDatasetOp::kDatasetType));
bytes_counter->IncrementBy(line_contents.size());
out_tensors->emplace_back(ctx->allocator({}), DT_STRING,
TensorShape({}));
out_tensors->back().scalar<tstring>()() = line_contents;
bytes_counter->IncrementBy(line_contents_str.size());
out_tensors->push_back(std::move(line_contents));
*end_of_sequence = false;
return Status::OK();
} else if (!errors::IsOutOfRange(s)) {

20
tensorflow/core/lib/io/buffered_inputstream.cc
View File

@ -21,17 +21,17 @@ namespace tensorflow {
namespace io {
BufferedInputStream::BufferedInputStream(InputStreamInterface* input_stream,
size_t buffer_size,
size_t buffer_bytes,
bool owns_input_stream)
: input_stream_(input_stream),
size_(buffer_size),
size_(buffer_bytes),
owns_input_stream_(owns_input_stream) {
buf_.reserve(size_);
}
BufferedInputStream::BufferedInputStream(RandomAccessFile* file,
size_t buffer_size)
: BufferedInputStream(new RandomAccessInputStream(file), buffer_size,
size_t buffer_bytes)
: BufferedInputStream(new RandomAccessInputStream(file), buffer_bytes,
true) {}
BufferedInputStream::~BufferedInputStream() {
@ -56,7 +56,9 @@ Status BufferedInputStream::FillBuffer() {
return s;
}
Status BufferedInputStream::ReadLineHelper(string* result, bool include_eol) {
template <typename StringType>
Status BufferedInputStream::ReadLineHelper(StringType* result,
bool include_eol) {
result->clear();
Status s;
while (true) {
@ -70,13 +72,13 @@ Status BufferedInputStream::ReadLineHelper(string* result, bool include_eol) {
char c = buf_[pos_++];
if (c == '\n') {
if (include_eol) {
*result += c;
result->append(1, c);
}
return Status::OK();
}
// We don't append '\r' to *result
if (c != '\r') {
*result += c;
result->append(1, c);
}
}
if (errors::IsOutOfRange(s) && !result->empty()) {
@ -202,6 +204,10 @@ Status BufferedInputStream::ReadLine(string* result) {
return ReadLineHelper(result, false);
}
Status BufferedInputStream::ReadLine(tstring* result) {
return ReadLineHelper(result, false);
}
string BufferedInputStream::ReadLineAsString() {
string result;
ReadLineHelper(&result, true).IgnoreError();

4
tensorflow/core/lib/io/buffered_inputstream.h
View File

@ -66,6 +66,7 @@ class BufferedInputStream : public InputStreamInterface {
// file, we return an OUT_OF_RANGE error. Otherwise, we return
// some other non-OK status.
tensorflow::Status ReadLine(string* result);
tensorflow::Status ReadLine(tstring* result);
// Returns one text line of data until end-of-file or a '\n' is read. The '\n'
// is included in the result.
@ -86,7 +87,8 @@ class BufferedInputStream : public InputStreamInterface {
private:
tensorflow::Status FillBuffer();
tensorflow::Status ReadLineHelper(string* result, bool include_eol);
template <typename StringType>
tensorflow::Status ReadLineHelper(StringType* result, bool include_eol);
InputStreamInterface* input_stream_; // not owned.
size_t size_; // buffer size.