experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[tf.data] Add metric to record file names read by tf.data Datasets.

Browse Source 
PiperOrigin-RevId: 355887516
Change-Id: Ic32e6cc9cd07e0ec24f77401cb3d2c6908a0a2cd
This commit is contained in:
Jiho Choi 2021-02-05 11:11:53 -08:00 committed by TensorFlower Gardener
parent 712dab5187
commit 3080eef5b0
5 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tensorflow/core/framework/metrics.cc
View File

@ -115,6 +115,10 @@ auto* tf_data_iterator_lifetime_counter = monitoring::Counter<0>::New(
auto* tf_data_optimization_counter = monitoring::Counter<1>::New( auto* tf_data_optimization_counter = monitoring::Counter<1>::New(
"/tensorflow/data/optimization", "tf.data optimization", "name"); "/tensorflow/data/optimization", "tf.data optimization", "name");
auto* tf_data_filename_counter = monitoring::Counter<2>::New(
"/tensorflow/data/filename", "The file name read by a tf.data Dataset.",
"name", "filename");
auto* parse_dense_feature_counter = monitoring::Counter<0>::New( auto* parse_dense_feature_counter = monitoring::Counter<0>::New(
"/tensorflow/data/dense_feature", "/tensorflow/data/dense_feature",
"The number of dense features parsed by ops for parsing tf.Example."); "The number of dense features parsed by ops for parsing tf.Example.");
@ -218,6 +222,10 @@ void RecordTFDataOptimization(const string& name, int64 num_changes) {
tf_data_optimization_counter->GetCell(name)->IncrementBy(num_changes); tf_data_optimization_counter->GetCell(name)->IncrementBy(num_changes);
} }
void RecordTFDataFilename(const string& name, const string& filename) {
tf_data_filename_counter->GetCell(name, filename)->IncrementBy(1);
}
void RecordParseDenseFeature(int64 num_features) { void RecordParseDenseFeature(int64 num_features) {
static auto* parse_dense_feature_counter_cell = static auto* parse_dense_feature_counter_cell =
parse_dense_feature_counter->GetCell(); parse_dense_feature_counter->GetCell();

5
tensorflow/core/framework/metrics.h
View File

@ -84,6 +84,11 @@ void RecordTFDataIteratorLifetime(uint64 duration_us);
// The `name` argument identifies the optimization (e.g. "noop_elimination"). // The `name` argument identifies the optimization (e.g. "noop_elimination").
void RecordTFDataOptimization(const string& name, int64 num_changes); void RecordTFDataOptimization(const string& name, int64 num_changes);
// Records the file name read by a tf.data Dataset.
//
// The `name` argument identifies the Dataset type (e.g. "TFRecordDataset").
void RecordTFDataFilename(const string& name, const string& filename);
// Records parsing of dense tensor features. // Records parsing of dense tensor features.
void RecordParseDenseFeature(int64 num_features); void RecordParseDenseFeature(int64 num_features);

1
tensorflow/core/kernels/data/fixed_length_record_dataset_op.cc
View File

@ -471,6 +471,7 @@ void FixedLengthRecordDatasetOp::MakeDataset(OpKernelContext* ctx,
filenames.reserve(filenames_tensor->NumElements()); filenames.reserve(filenames_tensor->NumElements());
for (int i = 0; i < filenames_tensor->NumElements(); ++i) { for (int i = 0; i < filenames_tensor->NumElements(); ++i) {
filenames.push_back(filenames_tensor->flat<tstring>()(i)); filenames.push_back(filenames_tensor->flat<tstring>()(i));
metrics::RecordTFDataFilename(kDatasetType, filenames[i]);
} }
int64 header_bytes = -1; int64 header_bytes = -1;

1
tensorflow/core/kernels/data/text_line_dataset_op.cc
View File

@ -275,6 +275,7 @@ void TextLineDatasetOp::MakeDataset(OpKernelContext* ctx,
filenames.reserve(filenames_tensor->NumElements()); filenames.reserve(filenames_tensor->NumElements());
for (int i = 0; i < filenames_tensor->NumElements(); ++i) { for (int i = 0; i < filenames_tensor->NumElements(); ++i) {
filenames.push_back(filenames_tensor->flat<tstring>()(i)); filenames.push_back(filenames_tensor->flat<tstring>()(i));
metrics::RecordTFDataFilename(kDatasetType, filenames[i]);
} }
*output = new Dataset(ctx, std::move(filenames), compression_type, *output = new Dataset(ctx, std::move(filenames), compression_type,

1
tensorflow/core/kernels/data/tf_record_dataset_op.cc
View File

@ -292,6 +292,7 @@ void TFRecordDatasetOp::MakeDataset(OpKernelContext* ctx,
filenames.push_back(filenames_tensor->flat<tstring>()(i)); filenames.push_back(filenames_tensor->flat<tstring>()(i));
is_gcs_fs &= absl::StartsWith(filenames[i], kGcsFsPrefix); is_gcs_fs &= absl::StartsWith(filenames[i], kGcsFsPrefix);
is_s3_fs &= absl::StartsWith(filenames[i], kS3FsPrefix); is_s3_fs &= absl::StartsWith(filenames[i], kS3FsPrefix);
metrics::RecordTFDataFilename(kDatasetType, filenames[i]);
} }
tstring compression_type; tstring compression_type;