experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Specialize suggestion for slow tf.data service.

Browse Source 
PiperOrigin-RevId: 339915378
Change-Id: I5675fdf910a3d8fd550d28a5382f0da20e450281
This commit is contained in:
Jiho Choi 2020-10-30 12:09:34 -07:00 committed by TensorFlower Gardener
parent 1b098d27d3
commit 9c3c3855bd
2 changed files with 14 additions and 1 deletions

14
tensorflow/core/profiler/convert/xplane_to_tf_data_stats.cc
View File

@ -296,6 +296,9 @@ std::string GetSuggestion(BottleneckType type) {
"data_performance#parallelizing_data_transformation"; "data_performance#parallelizing_data_transformation";
constexpr absl::string_view kTfGuideCacheLink = constexpr absl::string_view kTfGuideCacheLink =
"https://www.tensorflow.org/guide/data_performance#caching"; "https://www.tensorflow.org/guide/data_performance#caching";
constexpr absl::string_view kTfDataServiceLink =
"https://www.tensorflow.org/api_docs/python/tf/data/experimental/"
"service?version=nightly";
switch (type) { switch (type) {
case BottleneckType::kSlowSource: case BottleneckType::kSlowSource:
return absl::StrFormat( return absl::StrFormat(
@ -306,6 +309,14 @@ std::string GetSuggestion(BottleneckType type) {
"more details.<br/>", "more details.<br/>",
AnchorElement(kPlaybookSourceDatasetLink, "here"), AnchorElement(kPlaybookSourceDatasetLink, "here"),
AnchorElement(kTfGuideParallelDataExtractionLink, "here")); AnchorElement(kTfGuideParallelDataExtractionLink, "here"));
case BottleneckType::kSlowDataService:
return absl::StrFormat(
"1. Fetching data from tf.data service took a while. Profile the "
"tf.data service worker to analyze the issue further.<br/>"
"2. See %s for more details on tf.data service.<br/>"
"3. See %s for other suggestions.",
AnchorElement(kTfDataServiceLink, "this"),
AnchorElement(kPlaybookLink, "this"));
case BottleneckType::kSlowRemoteSource: case BottleneckType::kSlowRemoteSource:
return absl::StrFormat( return absl::StrFormat(
"1. The remote data source is slow. Profile its host to analyze the " "1. The remote data source is slow. Profile its host to analyze the "
@ -399,10 +410,11 @@ BottleneckType GetBottleneckType(absl::string_view bottleneck_iterator_name) {
{"TensorSlice", BottleneckType::kSlowSource}, {"TensorSlice", BottleneckType::kSlowSource},
{"Generator", BottleneckType::kSlowSource}, {"Generator", BottleneckType::kSlowSource},
{"SyntheticDatasetOp", BottleneckType::kSlowSource}, {"SyntheticDatasetOp", BottleneckType::kSlowSource},
// tf.data service.
{"DataService", BottleneckType::kSlowDataService},
// Read from remote memory. // Read from remote memory.
{"GuzzlerDataGuzzlerRemoteDataset", BottleneckType::kSlowRemoteSource}, {"GuzzlerDataGuzzlerRemoteDataset", BottleneckType::kSlowRemoteSource},
{"ReverbDataset", BottleneckType::kSlowRemoteSource}, {"ReverbDataset", BottleneckType::kSlowRemoteSource},
{"DatasetService", BottleneckType::kSlowRemoteSource},
{"DatasetSampleGame", BottleneckType::kSlowRemoteSource}, {"DatasetSampleGame", BottleneckType::kSlowRemoteSource},
{"Courier", BottleneckType::kSlowRemoteSource}, {"Courier", BottleneckType::kSlowRemoteSource},
{"ReverbEpisodeDataset", BottleneckType::kSlowRemoteSource}, {"ReverbEpisodeDataset", BottleneckType::kSlowRemoteSource},

1
tensorflow/core/profiler/convert/xplane_to_tf_data_stats.h
View File

@ -29,6 +29,7 @@ TF_CONST_INIT extern const int64 kSlowCallThresholdPs;
enum class BottleneckType { enum class BottleneckType {
kSlowSource, kSlowSource,
kSlowDataService,
kSlowRemoteSource, kSlowRemoteSource,
kSlowTransformationWithParallelVersion, kSlowTransformationWithParallelVersion,
kSlowTransformationWithoutParallelVersion, kSlowTransformationWithoutParallelVersion,