experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Reduce log verbosity and avoid spurious warning message for error aggregation and reporting.

Browse Source 
PiperOrigin-RevId: 247057740
This commit is contained in:
Jing Dong 2019-05-07 11:29:00 -07:00 committed by TensorFlower Gardener
parent 4259bfb42b
commit 9d19b1a1c6
2 changed files with 5 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
tensorflow/core/common_runtime/executor.cc
View File

@ -2231,14 +2231,10 @@ bool ExecutorState::NodeDone(const Status& s, const Node* node,
if (cancellation_manager_) {
// only log when the abort happens during the actual run time.
auto device_name = impl_->params_.device->name();
// Do not log OutOfRange errors as warnings because they are expected when
// Use VLOG instead of LOG(warning) because error status is expected when
// the executor is run under the grappler optimization phase or when
// iterating through a tf.data input pipeline.
if (!errors::IsOutOfRange(s)) {
LOG(WARNING) << "[" << device_name
<< "] Executor start aborting: " << s;
} else {
VLOG(1) << "[" << device_name << "] Executor start aborting: " << s;
}
VLOG(1) << "[" << device_name << "] Executor start aborting: " << s;
}
if (rendezvous_) {

10
tensorflow/core/distributed_runtime/master_session.cc
View File

@ -548,19 +548,13 @@ class RunManyGraphs {
bool cancel_issued_ GUARDED_BY(mu_) = false;
void ReportBadStatus(const Status& s) EXCLUSIVE_LOCKS_REQUIRED(mu_) {
// Start cancellation if we aren't already in an error state.
// TODO(jingdong): Change the following log to VLOG once the distributed
// error aggregation is stable.
LOG(INFO) << "Master received error status " << s;
VLOG(1) << "Master received error status " << s;
if (!cancel_issued_ && !StatusGroup::IsDerived(s)) {
// Only start cancelling other workers upon receiveing a non-derived
// error
cancel_issued_ = true;
// TODO(jingdong): Change the following log to VLOG once the distributed
// error aggregation feature is stable.
LOG(INFO)
<< "Master received error report. Cancelling remaining workers.";
VLOG(1) << "Master received error report. Cancelling remaining workers.";
for (Call& call : calls_) {
call.opts.StartCancel();
}