experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Disable cost tracking for async kernels.

Browse Source 
Async kernels can be modified from both the scheduling thread or a thread-pool,
making it difficult to safely update timing information from the scheduling
thread.

PiperOrigin-RevId: 226640910
This commit is contained in:
Russell Power 2018-12-22 18:12:58 -08:00 committed by TensorFlower Gardener
parent b27d50f234
commit 3785cffbdb
1 changed files with 9 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
tensorflow/core/common_runtime/executor.cc
View File

@ -135,13 +135,9 @@ struct EdgeInfo {
// Time the execution of kernels (in CPU cycles). Used to dynamically identify // Time the execution of kernels (in CPU cycles). Used to dynamically identify
// inexpensive kernels which can be dispatched inline. // inexpensive kernels which can be dispatched inline.
struct KernelTimer { struct KernelTimer {
uint64 start_cycles = 0; uint64 start_cycles = profile_utils::CpuUtils::GetCurrentClockCycle();
void Start() { uint64 ElapsedCycles() {
start_cycles = profile_utils::CpuUtils::GetCurrentClockCycle();
}
uint64 Stop() {
return profile_utils::CpuUtils::GetCurrentClockCycle() - start_cycles; return profile_utils::CpuUtils::GetCurrentClockCycle() - start_cycles;
} }
}; };
@ -1768,24 +1764,11 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) {
if (completed) Finish(); if (completed) Finish();
}; };
nodestats::SetOpStart(stats); nodestats::SetOpStart(stats);
KernelTimer timer;
if (item.kernel->IsExpensive()) {
timer.Start();
}
device->ComputeAsync(async, &state->ctx, done); device->ComputeAsync(async, &state->ctx, done);
// Update kernel cost based on time spent in the current thread.
if (item.kernel->IsExpensive()) {
async->UpdateCostEstimate(timer.Stop());
}
} else { } else {
// Synchronous computes. // Synchronous computes.
OpKernelContext ctx(&params, item.num_outputs); OpKernelContext ctx(&params, item.num_outputs);
nodestats::SetOpStart(stats); nodestats::SetOpStart(stats);
KernelTimer timer;
if (item.kernel->IsExpensive()) {
timer.Start();
}
if (TF_PREDICT_FALSE( if (TF_PREDICT_FALSE(
MightTrace(item, event_collector_, trace_using_annotations_))) { MightTrace(item, event_collector_, trace_using_annotations_))) {
@ -1812,11 +1795,13 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) {
} }
} else { } else {
// In the common case, avoid creating any tracing objects. // In the common case, avoid creating any tracing objects.
device->Compute(op_kernel, &ctx); if (op_kernel->IsExpensive()) {
} KernelTimer timer;
device->Compute(op_kernel, &ctx);
if (item.kernel->IsExpensive()) { op_kernel->UpdateCostEstimate(timer.ElapsedCycles());
op_kernel->UpdateCostEstimate(timer.Stop()); } else {
device->Compute(op_kernel, &ctx);
}
} }
nodestats::SetOpEnd(stats); nodestats::SetOpEnd(stats);