From 755a03b7f6b2a971ae38f4a77f879bde05f054ef Mon Sep 17 00:00:00 2001 From: Jiho Choi Date: Fri, 13 Nov 2020 16:20:17 -0800 Subject: [PATCH] Add the latency of the bottleneck iterator. PiperOrigin-RevId: 342359843 Change-Id: I7fc60232520a26b967053c579fc539ff177eeff5 --- .../convert/xplane_to_tf_data_stats.cc | 23 +++++++++++++------ .../convert/xplane_to_tf_data_stats_test.cc | 7 ++++++ .../profiler/protobuf/tf_data_stats.proto | 4 ++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/profiler/convert/xplane_to_tf_data_stats.cc b/tensorflow/core/profiler/convert/xplane_to_tf_data_stats.cc index 195afa3954e..0b278fccf3e 100644 --- a/tensorflow/core/profiler/convert/xplane_to_tf_data_stats.cc +++ b/tensorflow/core/profiler/convert/xplane_to_tf_data_stats.cc @@ -198,6 +198,7 @@ void SetBottleneckIteratorId(InputPipelineStat* input_pipeline_stat) { } } input_pipeline_stat->set_bottleneck_iterator_id(bottleneck_iterator_id); + input_pipeline_stat->set_bottleneck_iterator_latency_ps(max_self_time); } void ProcessInputPipelines( @@ -256,17 +257,20 @@ void SetBottleneckAnalysis(CombinedTfDataStats* combined_tf_data_stats) { InputPipeline(absl::string_view host_name, absl::string_view input_pipeline_name, int64 max_latency_ps, absl::string_view iterator_name, - absl::string_view iterator_long_name) + absl::string_view iterator_long_name, + int64 iterator_latency_ps) : host_name(host_name), input_pipeline_name(input_pipeline_name), max_latency_ps(max_latency_ps), iterator_name(iterator_name), - iterator_long_name(iterator_long_name) {} + iterator_long_name(iterator_long_name), + iterator_latency_ps(iterator_latency_ps) {} absl::string_view host_name; absl::string_view input_pipeline_name; int64 max_latency_ps; absl::string_view iterator_name; absl::string_view iterator_long_name; + int64 iterator_latency_ps; bool operator<(const InputPipeline& rhs) const { return max_latency_ps > rhs.max_latency_ps; @@ -286,12 +290,15 @@ void SetBottleneckAnalysis(CombinedTfDataStats* combined_tf_data_stats) { } // Choose the slowest execution trace of the input pipeline. // `input_pipeline_stats.stats` is already sorted so choose the first one. + const InputPipelineStat& input_pipeline_stat = + input_pipeline_stats.stats(0); const IteratorMetadata& metadata = tf_data_stats.iterator_metadata().at( - input_pipeline_stats.stats(0).bottleneck_iterator_id()); - slow_input_pipelines.emplace_back(host_name, - input_pipeline_stats.metadata().name(), - input_pipeline_stats.max_latency_ps(), - metadata.name(), metadata.long_name()); + input_pipeline_stat.bottleneck_iterator_id()); + slow_input_pipelines.emplace_back( + host_name, input_pipeline_stats.metadata().name(), + input_pipeline_stats.max_latency_ps(), metadata.name(), + metadata.long_name(), + input_pipeline_stat.bottleneck_iterator_latency_ps()); } } std::sort(slow_input_pipelines.begin(), slow_input_pipelines.end()); @@ -309,6 +316,8 @@ void SetBottleneckAnalysis(CombinedTfDataStats* combined_tf_data_stats) { bottleneck_analysis->set_iterator_long_name( input_pipeline.iterator_long_name.data(), input_pipeline.iterator_long_name.size()); + bottleneck_analysis->set_iterator_latency_ps( + input_pipeline.iterator_latency_ps); } } diff --git a/tensorflow/core/profiler/convert/xplane_to_tf_data_stats_test.cc b/tensorflow/core/profiler/convert/xplane_to_tf_data_stats_test.cc index 1066937cd80..3a450e66d97 100644 --- a/tensorflow/core/profiler/convert/xplane_to_tf_data_stats_test.cc +++ b/tensorflow/core/profiler/convert/xplane_to_tf_data_stats_test.cc @@ -85,6 +85,7 @@ TEST(XPlaneToTfDataStatsTest, HostInputPipeline) { max_latency_ps: 100000000 iterator_name: "Range" iterator_long_name: "Iterator::Prefetch::Range" + iterator_latency_ps: 80000000 suggestion: "See this for suggestions." } tf_data_stats: { @@ -119,6 +120,7 @@ TEST(XPlaneToTfDataStatsTest, HostInputPipeline) { num_slow_calls: 1 stats { bottleneck_iterator_id: 456 + bottleneck_iterator_latency_ps: 80000000 iterator_stats { key: 123, value: { @@ -144,6 +146,7 @@ TEST(XPlaneToTfDataStatsTest, HostInputPipeline) { } stats { bottleneck_iterator_id: 123 + bottleneck_iterator_latency_ps: 20000000 iterator_stats { key: 123, value: { @@ -243,6 +246,7 @@ TEST(XPlaneToTfDataStatsTest, DeviceInputPipeline) { num_slow_calls: 1 stats { bottleneck_iterator_id: 456 + bottleneck_iterator_latency_ps: 80000000 iterator_stats { key: 123, value: { @@ -268,6 +272,7 @@ TEST(XPlaneToTfDataStatsTest, DeviceInputPipeline) { } stats { bottleneck_iterator_id: 123 + bottleneck_iterator_latency_ps: 30000000 iterator_stats { key: 123, value: { @@ -338,6 +343,7 @@ TEST(XPlaneToTfDataStatsTest, MapAndBatch) { max_latency_ps: 100000000 iterator_name: "Range" iterator_long_name: "Iterator::MapAndBatch::Range" + iterator_latency_ps: 60000000 suggestion: "See this for suggestions." } tf_data_stats: { @@ -372,6 +378,7 @@ TEST(XPlaneToTfDataStatsTest, MapAndBatch) { num_slow_calls: 1 stats { bottleneck_iterator_id: 456 + bottleneck_iterator_latency_ps: 60000000 iterator_stats { key: 123, value: { diff --git a/tensorflow/core/profiler/protobuf/tf_data_stats.proto b/tensorflow/core/profiler/protobuf/tf_data_stats.proto index df203a7edeb..d9049e54e5e 100644 --- a/tensorflow/core/profiler/protobuf/tf_data_stats.proto +++ b/tensorflow/core/profiler/protobuf/tf_data_stats.proto @@ -45,6 +45,8 @@ message IteratorMetadata { message InputPipelineStat { // Id of the blocking iterator with the longest self time. int64 bottleneck_iterator_id = 2; + // Latency of the bottleneck iterator. + int64 bottleneck_iterator_latency_ps = 3; // Stats per iterator. map iterator_stats = 1; } @@ -101,6 +103,8 @@ message TfDataBottleneckAnalysis { string iterator_name = 4; // Long name of the bottleneck iterator. string iterator_long_name = 5; + // Latency of the bottleneck iterator. + int64 iterator_latency_ps = 7; // Suggestion to resolve the bottleneck. string suggestion = 6; }