experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

tfstreamz metric provider.

Browse Source 
PiperOrigin-RevId: 307760649
Change-Id: Icd8c74771d4854ff75fb57ad68b951bae9771356
This commit is contained in:
A. Unique TensorFlower 2020-04-22 00:13:00 -07:00 committed by TensorFlower Gardener
parent f09a9e344c
commit a51b060d16
5 changed files with 1 additions and 172 deletions

15
tensorflow/core/profiler/utils/BUILD
View File

@ -325,18 +325,3 @@ cc_library(
"@com_google_absl//absl/strings",
],
)
cc_library(
name = "tfstreamz_utils",
srcs = ["tfstreamz_utils.cc"],
hdrs = ["tfstreamz_utils.h"],
deps = [
":xplane_builder",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core/profiler/protobuf:xplane_proto_cc",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
],
)

105
tensorflow/core/profiler/utils/tfstreamz_utils.cc
View File

@ -1,105 +0,0 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/utils/tfstreamz_utils.h"
#include <memory>
#include "absl/memory/memory.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl/strings/substitute.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/gtl/map_util.h"
#include "tensorflow/core/lib/monitoring/collected_metrics.h"
#include "tensorflow/core/lib/monitoring/collection_registry.h"
#include "tensorflow/core/platform/errors.h"
namespace tensorflow {
namespace profiler {
namespace {
string ConstructXStatName(const string& name, const monitoring::Point& point) {
if (point.labels.empty()) {
return name;
}
return absl::Substitute(
"$0{$1}", name,
absl::StrJoin(point.labels, ", ",
[](string* out, const monitoring::Point::Label& label) {
absl::StrAppend(out, label.name, "=", label.value);
}));
}
string SerializePercentile(const monitoring::Percentiles& percentiles) {
return "";
}
} // namespace
Status SerializeToXPlane(const std::vector<TfStreamzSnapshot>& snapshots,
XPlane* plane) {
XPlaneBuilder xplane(plane);
XLineBuilder line = xplane.GetOrCreateLine(0); // This plane has single line.
// For each snapshot, create a virtual event.
for (const auto& snapshot : snapshots) {
XEventMetadata* event_metadata =
xplane.GetOrCreateEventMetadata("TFStreamz Snapshot");
XEventBuilder xevent = line.AddEvent(*event_metadata);
xevent.SetTimestampNs(snapshot.start_time_ns);
xevent.SetEndTimestampNs(snapshot.end_time_ns);
auto& metric_descriptor_map = snapshot.metrics->metric_descriptor_map;
for (const auto& point_set : snapshot.metrics->point_set_map) {
const string& metric_name = point_set.first;
// Each metrics have multiple points corresponding to different labels.
for (const auto& point : point_set.second->points) {
// Generates one KPI metric for each point.
string stat_name = ConstructXStatName(metric_name, *point);
auto* metadata = xplane.GetOrCreateStatMetadata(stat_name);
auto it = metric_descriptor_map.find(metric_name);
if (it != metric_descriptor_map.end()) {
metadata->set_description(it->second->description);
}
switch (point->value_type) {
case monitoring::ValueType::kInt64:
xevent.AddStatValue(*metadata, point->int64_value);
break;
case monitoring::ValueType::kBool:
xevent.AddStatValue(*metadata, point->bool_value);
break;
case monitoring::ValueType::kString:
xevent.AddStatValue(*metadata, point->string_value);
break;
case monitoring::ValueType::kHistogram:
xevent.AddStatValue(*metadata,
point->histogram_value.SerializeAsString(),
/*is_bytes=*/true);
break;
case monitoring::ValueType::kPercentiles:
// TODO(jiesun): define a proto to hold monitoring::Percentiles.
xevent.AddStatValue(*metadata,
SerializePercentile(point->percentiles_value),
/*is_bytes=*/true);
break;
}
}
}
}
return Status::OK();
}
} // namespace profiler
} // namespace tensorflow

39
tensorflow/core/profiler/utils/tfstreamz_utils.h
View File

@ -1,39 +0,0 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_UTILS_TFSTREAMZ_UTILS_H_
#define TENSORFLOW_CORE_PROFILER_UTILS_TFSTREAMZ_UTILS_H_
#include "tensorflow/core/lib/monitoring/collected_metrics.h"
#include "tensorflow/core/lib/monitoring/collection_registry.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/profiler/protobuf/xplane.pb.h"
#include "tensorflow/core/profiler/utils/xplane_builder.h"
namespace tensorflow {
namespace profiler {
struct TfStreamzSnapshot {
std::unique_ptr<monitoring::CollectedMetrics> metrics;
uint64 start_time_ns; // time before collection.
uint64 end_time_ns; // time after collection.
};
Status SerializeToXPlane(const std::vector<TfStreamzSnapshot>& snapshots,
XPlane* plane);
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_UTILS_TFSTREAMZ_UTILS_H_

7
tensorflow/core/profiler/utils/xplane_schema.cc
View File

@ -26,16 +26,11 @@ const absl::string_view kHostThreads = "/host:CPU";
const absl::string_view kGpuPlanePrefix = "/device:GPU:";
const absl::string_view kCuptiDriverApiPlaneName = "/host:CUPTI";
const absl::string_view kMetadataPlane = "/host:metadata";
const absl::string_view kTFStreamzPlane = "/host:tfstreamz";
const int32 kHostPlaneId = 49;
const int32 kGpuPlaneBaseId = 0;
const int32 kCuptiDriverApiPlaneId = 50;
const int32 kMetadataPlaneId = 99;
const int32 kTFStreamzPlaneId = 98;
const int32 kThreadGroupMinPlaneId = kCuptiDriverApiPlaneId + 1;
const int32 kThreadGroupMaxPlaneId = kTFStreamzPlaneId - 1;
const int32 kMetadataPlaneId = 51;
namespace {

7
tensorflow/core/profiler/utils/xplane_schema.h
View File

@ -33,8 +33,6 @@ ABSL_CONST_INIT extern const absl::string_view kGpuPlanePrefix;
ABSL_CONST_INIT extern const absl::string_view kCuptiDriverApiPlaneName;
// Name of XPlane that contains profile metadata such as XLA debug info.
ABSL_CONST_INIT extern const absl::string_view kMetadataPlane;
// Name of XPlane that contains kpi related metrics.
ABSL_CONST_INIT extern const absl::string_view kTFStreamzPlane;
// Id of XPlane that contains TraceMe events.
ABSL_CONST_INIT extern const int32 kHostPlaneId;
@ -45,11 +43,6 @@ ABSL_CONST_INIT extern const int32 kGpuPlaneBaseId;
ABSL_CONST_INIT extern const int32 kCuptiDriverApiPlaneId;
// Id of XPlane that contains profile metadata such as XLA debug info.
ABSL_CONST_INIT extern const int32 kMetadataPlaneId;
// Id of XPlane that contains kpi related metrics.
ABSL_CONST_INIT extern const int32 kTFStreamzPlaneId;
ABSL_CONST_INIT extern const int32 kThreadGroupMinPlaneId;
ABSL_CONST_INIT extern const int32 kThreadGroupMaxPlaneId;
// Interesting event types (i.e., TraceMe names).
enum HostEventType {