experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[tf.data] Record the number of times tf.data experiment applied to input pipelines.

Browse Source 
PiperOrigin-RevId: 325345128
Change-Id: I8de30c47f681a6f41e25e5ade4460f20a9d7bb5d
This commit is contained in:
Jay Shi 2020-08-06 17:32:29 -07:00 committed by TensorFlower Gardener
parent bcf052b88c
commit d9ea505110
4 changed files with 27 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
tensorflow/core/framework/metrics.cc
View File

@ -80,6 +80,11 @@ auto* tf_data_bytes_fetched_counter = monitoring::Counter<0>::New(
auto* tf_data_elements_counter = monitoring::Counter<1>::New( auto* tf_data_elements_counter = monitoring::Counter<1>::New(
"/tensorflow/data/elements", "tf.data elements", "name"); "/tensorflow/data/elements", "tf.data elements", "name");
auto* tf_data_experiment_counter = monitoring::Counter<1>::New(
"/tensorflow/data/experiment",
"The number of times tf.data experiment is applied to input pipelines.",
"name");
auto* tf_data_fingerprint_counter = monitoring::Counter<1>::New( auto* tf_data_fingerprint_counter = monitoring::Counter<1>::New(
"/tensorflow/data/fingerprint", "tf.data fingerprint", "name"); "/tensorflow/data/fingerprint", "tf.data fingerprint", "name");
@ -179,6 +184,10 @@ void RecordTFDataBytesFetched(int64 num_bytes) {
tf_data_bytes_fetched_counter->GetCell()->IncrementBy(num_bytes); tf_data_bytes_fetched_counter->GetCell()->IncrementBy(num_bytes);
} }
void RecordTFDataExperiment(const string& name) {
tf_data_experiment_counter->GetCell(name)->IncrementBy(1);
}
void RecordTFDataFingerprint(const string& name) { void RecordTFDataFingerprint(const string& name) {
tf_data_fingerprint_counter->GetCell(name)->IncrementBy(1); tf_data_fingerprint_counter->GetCell(name)->IncrementBy(1);
} }

3
tensorflow/core/framework/metrics.h
View File

@ -56,6 +56,9 @@ monitoring::CounterCell* GetTFDataElementsCounter(const string& name);
// Records the number of bytes fetched from tf.data.Dataset iterator. // Records the number of bytes fetched from tf.data.Dataset iterator.
void RecordTFDataBytesFetched(int64 num_bytes); void RecordTFDataBytesFetched(int64 num_bytes);
// Records the number of times tf.data experiment is applied to input pipelines.
void RecordTFDataExperiment(const string& name);
// Records the time spent in ItertatorResource::GetNext() in microseconds. // Records the time spent in ItertatorResource::GetNext() in microseconds.
void RecordTFDataGetNextDuration(uint64 duration_us); void RecordTFDataGetNextDuration(uint64 duration_us);

11
tensorflow/core/kernels/data/dataset_utils.cc
View File

@ -1018,17 +1018,6 @@ std::vector<tstring> SelectOptimizations(
} }
} }
// Log the experiments that will be applied.
if (VLOG_IS_ON(1)) {
for (auto& pair : live_experiments) {
string experiment = pair.first;
if (std::find(optimizations_set.begin(), optimizations_set.end(),
experiment) != optimizations_set.end()) {
VLOG(1) << "The experiment \"" << experiment << "\" is applied.";
}
}
}
std::vector<tstring> optimizations; std::vector<tstring> optimizations;
optimizations.insert(optimizations.end(), optimizations_set.begin(), optimizations.insert(optimizations.end(), optimizations_set.begin(),
optimizations_set.end()); optimizations_set.end());

15
tensorflow/core/kernels/data/optimize_dataset_op.cc
View File

@ -101,6 +101,21 @@ void OptimizeDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input,
job_name, opt_ins_raw, opt_outs_raw, live_experiments, job_name, opt_ins_raw, opt_outs_raw, live_experiments,
optimizations_enabled, optimizations_disabled, optimizations_default, optimizations_enabled, optimizations_disabled, optimizations_default,
hash_func); hash_func);
// Log the experiments that will be applied.
if (!live_experiments.empty() && VLOG_IS_ON(1)) {
VLOG(1) << "The input pipeline is subject to tf.data experiment. "
"Please see `go/tf-data-experiments` for more details.";
for (auto& pair : live_experiments) {
string experiment = pair.first;
if (std::find(optimizations.begin(), optimizations.end(),
experiment) != optimizations.end()) {
VLOG(1) << "The experiment \"" << experiment << "\" is applied.";
metrics::RecordTFDataExperiment(experiment);
}
}
}
} }
} }