experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add the latency of the bottleneck iterator.

Browse Source 
PiperOrigin-RevId: 342359843
Change-Id: I7fc60232520a26b967053c579fc539ff177eeff5
This commit is contained in:
Jiho Choi 2020-11-13 16:20:17 -08:00 committed by TensorFlower Gardener
parent a3b0b2bf41
commit 755a03b7f6
3 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
tensorflow/core/profiler/convert/xplane_to_tf_data_stats.cc
View File

@ -198,6 +198,7 @@ void SetBottleneckIteratorId(InputPipelineStat* input_pipeline_stat) {
} }
} }
input_pipeline_stat->set_bottleneck_iterator_id(bottleneck_iterator_id); input_pipeline_stat->set_bottleneck_iterator_id(bottleneck_iterator_id);
input_pipeline_stat->set_bottleneck_iterator_latency_ps(max_self_time);
} }
void ProcessInputPipelines( void ProcessInputPipelines(
@ -256,17 +257,20 @@ void SetBottleneckAnalysis(CombinedTfDataStats* combined_tf_data_stats) {
InputPipeline(absl::string_view host_name, InputPipeline(absl::string_view host_name,
absl::string_view input_pipeline_name, int64 max_latency_ps, absl::string_view input_pipeline_name, int64 max_latency_ps,
absl::string_view iterator_name, absl::string_view iterator_name,
absl::string_view iterator_long_name) absl::string_view iterator_long_name,
int64 iterator_latency_ps)
: host_name(host_name), : host_name(host_name),
input_pipeline_name(input_pipeline_name), input_pipeline_name(input_pipeline_name),
max_latency_ps(max_latency_ps), max_latency_ps(max_latency_ps),
iterator_name(iterator_name), iterator_name(iterator_name),
iterator_long_name(iterator_long_name) {} iterator_long_name(iterator_long_name),
iterator_latency_ps(iterator_latency_ps) {}
absl::string_view host_name; absl::string_view host_name;
absl::string_view input_pipeline_name; absl::string_view input_pipeline_name;
int64 max_latency_ps; int64 max_latency_ps;
absl::string_view iterator_name; absl::string_view iterator_name;
absl::string_view iterator_long_name; absl::string_view iterator_long_name;
int64 iterator_latency_ps;
bool operator<(const InputPipeline& rhs) const { bool operator<(const InputPipeline& rhs) const {
return max_latency_ps > rhs.max_latency_ps; return max_latency_ps > rhs.max_latency_ps;
@ -286,12 +290,15 @@ void SetBottleneckAnalysis(CombinedTfDataStats* combined_tf_data_stats) {
} }
// Choose the slowest execution trace of the input pipeline. // Choose the slowest execution trace of the input pipeline.
// `input_pipeline_stats.stats` is already sorted so choose the first one. // `input_pipeline_stats.stats` is already sorted so choose the first one.
const InputPipelineStat& input_pipeline_stat =
input_pipeline_stats.stats(0);
const IteratorMetadata& metadata = tf_data_stats.iterator_metadata().at( const IteratorMetadata& metadata = tf_data_stats.iterator_metadata().at(
input_pipeline_stats.stats(0).bottleneck_iterator_id()); input_pipeline_stat.bottleneck_iterator_id());
slow_input_pipelines.emplace_back(host_name, slow_input_pipelines.emplace_back(
input_pipeline_stats.metadata().name(), host_name, input_pipeline_stats.metadata().name(),
input_pipeline_stats.max_latency_ps(), input_pipeline_stats.max_latency_ps(), metadata.name(),
metadata.name(), metadata.long_name()); metadata.long_name(),
input_pipeline_stat.bottleneck_iterator_latency_ps());
} }
} }
std::sort(slow_input_pipelines.begin(), slow_input_pipelines.end()); std::sort(slow_input_pipelines.begin(), slow_input_pipelines.end());
@ -309,6 +316,8 @@ void SetBottleneckAnalysis(CombinedTfDataStats* combined_tf_data_stats) {
bottleneck_analysis->set_iterator_long_name( bottleneck_analysis->set_iterator_long_name(
input_pipeline.iterator_long_name.data(), input_pipeline.iterator_long_name.data(),
input_pipeline.iterator_long_name.size()); input_pipeline.iterator_long_name.size());
bottleneck_analysis->set_iterator_latency_ps(
input_pipeline.iterator_latency_ps);
} }
} }

7
tensorflow/core/profiler/convert/xplane_to_tf_data_stats_test.cc
View File

@ -85,6 +85,7 @@ TEST(XPlaneToTfDataStatsTest, HostInputPipeline) {
max_latency_ps: 100000000 max_latency_ps: 100000000
iterator_name: "Range" iterator_name: "Range"
iterator_long_name: "Iterator::Prefetch::Range" iterator_long_name: "Iterator::Prefetch::Range"
iterator_latency_ps: 80000000
suggestion: "See <a href=\"https://www.tensorflow.org/guide/data_performance_analysis\" target=\"_blank\">this</a> for suggestions." suggestion: "See <a href=\"https://www.tensorflow.org/guide/data_performance_analysis\" target=\"_blank\">this</a> for suggestions."
} }
tf_data_stats: { tf_data_stats: {
@ -119,6 +120,7 @@ TEST(XPlaneToTfDataStatsTest, HostInputPipeline) {
num_slow_calls: 1 num_slow_calls: 1
stats { stats {
bottleneck_iterator_id: 456 bottleneck_iterator_id: 456
bottleneck_iterator_latency_ps: 80000000
iterator_stats { iterator_stats {
key: 123, key: 123,
value: { value: {
@ -144,6 +146,7 @@ TEST(XPlaneToTfDataStatsTest, HostInputPipeline) {
} }
stats { stats {
bottleneck_iterator_id: 123 bottleneck_iterator_id: 123
bottleneck_iterator_latency_ps: 20000000
iterator_stats { iterator_stats {
key: 123, key: 123,
value: { value: {
@ -243,6 +246,7 @@ TEST(XPlaneToTfDataStatsTest, DeviceInputPipeline) {
num_slow_calls: 1 num_slow_calls: 1
stats { stats {
bottleneck_iterator_id: 456 bottleneck_iterator_id: 456
bottleneck_iterator_latency_ps: 80000000
iterator_stats { iterator_stats {
key: 123, key: 123,
value: { value: {
@ -268,6 +272,7 @@ TEST(XPlaneToTfDataStatsTest, DeviceInputPipeline) {
} }
stats { stats {
bottleneck_iterator_id: 123 bottleneck_iterator_id: 123
bottleneck_iterator_latency_ps: 30000000
iterator_stats { iterator_stats {
key: 123, key: 123,
value: { value: {
@ -338,6 +343,7 @@ TEST(XPlaneToTfDataStatsTest, MapAndBatch) {
max_latency_ps: 100000000 max_latency_ps: 100000000
iterator_name: "Range" iterator_name: "Range"
iterator_long_name: "Iterator::MapAndBatch::Range" iterator_long_name: "Iterator::MapAndBatch::Range"
iterator_latency_ps: 60000000
suggestion: "See <a href=\"https://www.tensorflow.org/guide/data_performance_analysis\" target=\"_blank\">this</a> for suggestions." suggestion: "See <a href=\"https://www.tensorflow.org/guide/data_performance_analysis\" target=\"_blank\">this</a> for suggestions."
} }
tf_data_stats: { tf_data_stats: {
@ -372,6 +378,7 @@ TEST(XPlaneToTfDataStatsTest, MapAndBatch) {
num_slow_calls: 1 num_slow_calls: 1
stats { stats {
bottleneck_iterator_id: 456 bottleneck_iterator_id: 456
bottleneck_iterator_latency_ps: 60000000
iterator_stats { iterator_stats {
key: 123, key: 123,
value: { value: {

4
tensorflow/core/profiler/protobuf/tf_data_stats.proto
View File

@ -45,6 +45,8 @@ message IteratorMetadata {
message InputPipelineStat { message InputPipelineStat {
// Id of the blocking iterator with the longest self time. // Id of the blocking iterator with the longest self time.
int64 bottleneck_iterator_id = 2; int64 bottleneck_iterator_id = 2;
// Latency of the bottleneck iterator.
int64 bottleneck_iterator_latency_ps = 3;
// Stats per iterator. // Stats per iterator.
map<int64, IteratorStat> iterator_stats = 1; map<int64, IteratorStat> iterator_stats = 1;
} }
@ -101,6 +103,8 @@ message TfDataBottleneckAnalysis {
string iterator_name = 4; string iterator_name = 4;
// Long name of the bottleneck iterator. // Long name of the bottleneck iterator.
string iterator_long_name = 5; string iterator_long_name = 5;
// Latency of the bottleneck iterator.
int64 iterator_latency_ps = 7;
// Suggestion to resolve the bottleneck. // Suggestion to resolve the bottleneck.
string suggestion = 6; string suggestion = 6;
} }