experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add an XSpace to TraceEvents converter and unittest

Browse Source 
PiperOrigin-RevId: 290126021
Change-Id: Idbafefe96ade65aa74fd962124b31a2debe16b62
This commit is contained in:
A. Unique TensorFlower 2020-01-16 12:58:49 -08:00 committed by TensorFlower Gardener
parent 4097134b29
commit fd2cd3e10e
7 changed files with 250 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tensorflow/core/platform/env_time.h
View File

@ -29,6 +29,7 @@ class EnvTime {
static constexpr uint64 kMicrosToNanos = 1000ULL;
static constexpr uint64 kMillisToMicros = 1000ULL;
static constexpr uint64 kMillisToNanos = 1000ULL * 1000ULL;
static constexpr uint64 kNanosToPicos = 1000ULL;
static constexpr uint64 kSecondsToMillis = 1000ULL;
static constexpr uint64 kSecondsToMicros = 1000ULL * 1000ULL;
static constexpr uint64 kSecondsToNanos = 1000ULL * 1000ULL * 1000ULL;

34
tensorflow/core/profiler/convert/BUILD
View File

@ -1,3 +1,5 @@
load("//tensorflow:tensorflow.bzl", "tf_cc_test")
package(
default_visibility = ["//tensorflow/core/profiler:internal"],
licenses = ["notice"], # Apache 2.0
@ -171,3 +173,35 @@ cc_library(
"//tensorflow/core/profiler/utils:xplane_visitor",
],
)
cc_library(
name = "xplane_to_trace_events",
srcs = ["xplane_to_trace_events.cc"],
hdrs = ["xplane_to_trace_events.h"],
deps = [
"//tensorflow/core:lib",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core/profiler/protobuf:xplane_proto_cc",
"//tensorflow/core/profiler/utils:xplane_schema",
"//tensorflow/core/profiler/utils:xplane_visitor",
"@com_google_absl//absl/strings",
],
)
tf_cc_test(
name = "xplane_to_trace_events_test",
size = "small",
srcs = ["xplane_to_trace_events_test.cc"],
deps = [
":xplane_to_trace_events",
"//tensorflow/core:lib",
"//tensorflow/core:lib_internal",
"//tensorflow/core:protos_all_cc",
"//tensorflow/core:test",
"//tensorflow/core:test_main",
"//tensorflow/core:testlib",
"//tensorflow/core/profiler/utils:xplane_builder",
"//tensorflow/core/profiler/utils:xplane_schema",
"//tensorflow/core/profiler/utils:xplane_utils",
],
)

87
tensorflow/core/profiler/convert/xplane_to_trace_events.cc Normal file
View File

@ -0,0 +1,87 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/convert/xplane_to_trace_events.h"
#include "tensorflow/core/platform/env_time.h"
#include "tensorflow/core/profiler/utils/xplane_schema.h"
#include "tensorflow/core/profiler/utils/xplane_visitor.h"
namespace tensorflow {
namespace profiler {
namespace {
// Given a node_name in the format "op_name:op_type", returns the "op_type".
// If the "op_type" is missing, returns the node_name.
// This is done so all ops with the same type appear in the same color in trace
// viewer.
inline std::string EventName(absl::string_view node_name) {
std::vector<absl::string_view> parts = absl::StrSplit(node_name, ':');
return string(parts.back());
}
Device BuildDeviceAndResource(const XPlaneVisitor& plane) {
Device device;
device.set_name(std::string(plane.Name()));
device.set_device_id(plane.Id());
plane.ForEachLine([&](const XLineVisitor& line) {
Resource resource;
resource.set_resource_id(line.Id());
resource.set_name(std::string(line.Name()));
(*device.mutable_resources())[line.Id()] = resource;
});
return device;
}
} // namespace
void ConvertXSpaceToTraceEvents(uint64 profile_start_time_ns,
uint64 profile_end_time_ns,
const XSpace& xspace, Trace* trace) {
auto* trace_devices = trace->mutable_devices();
for (const auto& raw_plane : xspace.planes()) {
XPlaneVisitor xplane(&raw_plane);
// Convert devices and resources.
int64 device_id = xplane.Id();
(*trace_devices)[device_id] = BuildDeviceAndResource(xplane);
// Convert events.
xplane.ForEachLine([&](const XLineVisitor& xline) {
int64 resource_id = xline.Id(); // Either thread id or CUDA stream id.
xline.ForEachEvent([&](const XEventVisitor& xevent) {
if (xevent.TimestampNs() < profile_start_time_ns ||
xevent.TimestampNs() + xevent.DurationNs() > profile_end_time_ns) {
return;
}
auto* event = trace->add_trace_events();
auto& args = *event->mutable_args();
event->set_device_id(device_id);
event->set_resource_id(resource_id);
event->set_name(EventName(xevent.Name()));
event->set_timestamp_ps((xevent.TimestampNs() - profile_start_time_ns) *
EnvTime::kNanosToPicos);
event->set_duration_ps(xevent.DurationNs() * EnvTime::kNanosToPicos);
xevent.ForEachStat([&](const XStatVisitor& stat) {
if (stat.ValueCase() == XStat::VALUE_NOT_SET) return;
args[std::string(stat.Name())] = stat.ToString();
});
});
});
}
}
} // namespace profiler
} // namespace tensorflow

34
tensorflow/core/profiler/convert/xplane_to_trace_events.h Normal file
View File

@ -0,0 +1,34 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_TRACE_EVENTS_H_
#define TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_TRACE_EVENTS_H_
#include "absl/strings/str_split.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/profiler/protobuf/xplane.pb.h"
#include "tensorflow/core/protobuf/trace_events.pb.h"
namespace tensorflow {
namespace profiler {
void ConvertXSpaceToTraceEvents(uint64 profile_start_time_ns,
uint64 profile_end_time_ns,
const XSpace& xspace, Trace* trace);
} // namespace profiler
} // namespace tensorflow
#endif // TENSORFLOW_CORE_PROFILER_CONVERT_XPLANE_TO_TRACE_EVENTS_H_

77
tensorflow/core/profiler/convert/xplane_to_trace_events_test.cc Normal file
View File

@ -0,0 +1,77 @@
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/profiler/convert/xplane_to_trace_events.h"
#include "tensorflow/core/platform/test.h"
#include "tensorflow/core/profiler/utils/xplane_builder.h"
#include "tensorflow/core/profiler/utils/xplane_schema.h"
namespace tensorflow {
namespace profiler {
namespace {
void CreateXSpace(XSpace* space) {
XPlaneBuilder host_plane(space->add_planes());
XPlaneBuilder device_plane(space->add_planes());
host_plane.SetName("cpu");
host_plane.SetId(0);
XLineBuilder thread1 = host_plane.GetOrCreateLine(10);
thread1.SetName("thread1");
XEventBuilder event1 =
thread1.AddEvent(*host_plane.GetOrCreateEventMetadata("event1"));
event1.SetTimestampNs(150000);
event1.SetDurationNs(10000);
event1.ParseAndAddStatValue(*host_plane.GetOrCreateStatMetadata("tf_op"),
"Relu");
XLineBuilder thread2 = host_plane.GetOrCreateLine(20);
thread2.SetName("thread2");
XEventBuilder event2 =
thread2.AddEvent(*host_plane.GetOrCreateEventMetadata("event2"));
event2.SetTimestampNs(160000);
event2.SetDurationNs(10000);
event2.ParseAndAddStatValue(*host_plane.GetOrCreateStatMetadata("tf_op"),
"Conv2D");
device_plane.SetName("gpu:0");
device_plane.SetId(1);
XLineBuilder stream1 = device_plane.GetOrCreateLine(30);
stream1.SetName("gpu stream 1");
XEventBuilder event3 =
stream1.AddEvent(*device_plane.GetOrCreateEventMetadata("kernel1"));
event3.SetTimestampNs(180000);
event3.SetDurationNs(10000);
event3.ParseAndAddStatValue(
*device_plane.GetOrCreateStatMetadata("correlation id"), "55");
}
TEST(ConvertXPlaneToTraceEvents, Convert) {
XSpace xspace;
CreateXSpace(&xspace);
Trace trace;
ConvertXSpaceToTraceEvents(/*profile_start_time_ns*/ 100000,
/*profile_end_time_ns*/ 200000, xspace, &trace);
ASSERT_EQ(trace.devices_size(), 2);
EXPECT_EQ(trace.devices().at(0).resources_size(), 2);
EXPECT_EQ(trace.devices().at(1).resources_size(), 1);
EXPECT_EQ(trace.trace_events_size(), 3);
}
} // namespace
} // namespace profiler
} // namespace tensorflow

15
tensorflow/core/profiler/utils/xplane_visitor.cc
View File

@ -25,6 +25,21 @@ XStatVisitor::XStatVisitor(const XPlaneVisitor* plane, const XStat* stat)
metadata_(plane->GetStatMetadata(stat->metadata_id())),
type_(plane->GetStatType(stat->metadata_id())) {}
std::string XStatVisitor::ToString() const {
switch (stat_->value_case()) {
case XStat::kInt64Value:
return absl::StrCat(stat_->int64_value());
case XStat::kUint64Value:
return absl::StrCat(stat_->uint64_value());
case XStat::kDoubleValue:
return absl::StrCat(stat_->double_value());
case XStat::kStrValue:
return stat_->str_value();
case XStat::VALUE_NOT_SET:
return "";
}
}
XEventVisitor::XEventVisitor(const XPlaneVisitor* plane, const XLine* line,
const XEvent* event)
: XStatsOwner<XEvent>(plane, event),

2
tensorflow/core/profiler/utils/xplane_visitor.h
View File

@ -56,6 +56,8 @@ class XStatVisitor {
const XStat& RawStat() const { return *stat_; }
std::string ToString() const;
private:
const XStat* stat_;
const XStatMetadata* metadata_;