experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Trace CUPTI synchronization events.

Browse Source 
- cuEventSynchronize
 - cuStreamWaitEvent
 - cuStreamSynchronize
 - cuCtxSynchronize

PiperOrigin-RevId: 353984518
Change-Id: Ib3a5a1f50993248ae71f32baa54390fe8b29d298
This commit is contained in:
Justin Lebar 2021-01-26 17:11:49 -08:00 committed by TensorFlower Gardener
parent 470e4535d8
commit 747ca958fa
1 changed files with 35 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
tensorflow/core/profiler/internal/gpu/cupti_tracer.cc
View File

@ -776,6 +776,36 @@ void AddMemsetActivityEvent(CuptiTraceCollector *collector,
collector->AddEvent(std::move(event)); collector->AddEvent(std::move(event));
} }
void AddSynchronizationActivityEvent(
CuptiTraceCollector *collector, const CUpti_ActivitySynchronization *sync) {
CuptiTracerEvent event{};
event.type = CuptiTracerEventType::Generic;
event.source = CuptiTracerEventSource::Activity;
switch (sync->type) {
case CUPTI_ACTIVITY_SYNCHRONIZATION_TYPE_EVENT_SYNCHRONIZE:
event.name = "cuEventSynchronize";
break;
case CUPTI_ACTIVITY_SYNCHRONIZATION_TYPE_STREAM_WAIT_EVENT:
event.name = "cuStreamWaitEvent";
break;
case CUPTI_ACTIVITY_SYNCHRONIZATION_TYPE_STREAM_SYNCHRONIZE:
event.name = "cuStreamSynchronize";
break;
case CUPTI_ACTIVITY_SYNCHRONIZATION_TYPE_CONTEXT_SYNCHRONIZE:
event.name = "cuCtxSynchronize";
break;
default:
event.name = "unknown synchronization event";
break;
}
event.start_time_ns = sync->start;
event.end_time_ns = std::max(sync->end, sync->start + 1);
event.correlation_id = sync->correlationId;
event.context_id = sync->contextId;
VLOG(5) << "Cuda activity " << event.name;
collector->AddEvent(std::move(event));
}
// This hook uses cupti activity api to measure device side activities. // This hook uses cupti activity api to measure device side activities.
class CuptiDriverApiHookWithActivityApi : public CuptiDriverApiHook { class CuptiDriverApiHookWithActivityApi : public CuptiDriverApiHook {
public: public:
@ -1901,6 +1931,11 @@ Status CuptiTracer::ProcessActivityBuffer(CUcontext context, uint32_t stream_id,
AddMemsetActivityEvent( AddMemsetActivityEvent(
collector_, reinterpret_cast<CUpti_ActivityMemset *>(record)); collector_, reinterpret_cast<CUpti_ActivityMemset *>(record));
break; break;
case CUPTI_ACTIVITY_KIND_SYNCHRONIZATION:
AddSynchronizationActivityEvent(
collector_,
reinterpret_cast<CUpti_ActivitySynchronization *>(record));
break;
default: default:
LOG(ERROR) << "Activity type " << record->kind << " not supported."; LOG(ERROR) << "Activity type " << record->kind << " not supported.";
break; break;