experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make error message when explicitly adding an invalid device more clear.

Browse Source 
Before, you would get something like:

`InvalidArgumentError: Cannot assign a device to node 'save/RestoreV2_45': Could not satisfy explicit device specification '/job:ps/task:1/device:CPU:0' because no devices matching that specification are registered in this process; available devices: /job:localhost/replica:0/task:0/cpu:0 [[Node: save/RestoreV2_45 = RestoreV2[dtypes=[DT_FLOAT], _device="/job:ps/task:1/device:CPU:0"](save/Const, save/RestoreV2_45/tensor_names, save/RestoreV2_45/shape_and_slices)]] Caused by op u'save/RestoreV2_45', defined`

And now this reads:

`InvalidArgumentError: Cannot assign a device for operation 'save/RestoreV2_45':
Operation was explicitly assigned to '/job:ps/task:1/device:CPU:0'
but available devices are [ /job:localhost/replica:0/task:0/cpu:0 ]. Make
sure the device specification refers to a valid device.

This drops the additional debug_info because this condition is tightly
scoped to lack of devices, and so node information is unnecessary.
Change: 154379360
This commit is contained in:
Vijay Vasudevan 2017-04-26 19:02:10 -08:00 committed by TensorFlower Gardener
parent be43153b21
commit 95ca363c6c
4 changed files with 17 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tensorflow/core/common_runtime/simple_placer.cc
View File

@ -341,11 +341,10 @@ class ColocationGraph {
std::sort(device_names.begin(), device_names.end()); std::sort(device_names.begin(), device_names.end());
return errors::InvalidArgument( return errors::InvalidArgument(
"Could not satisfy explicit device specification '", "Operation was explicitly assigned to ", node->def().device(),
node->def().device(), " but available devices are [ ",
"' because no devices matching that specification " str_util::Join(device_names, ", "), " ]. Make sure ",
"are registered in this process; available devices: ", "the device specification refers to a valid device.");
str_util::Join(device_names, ", "), debug_info);
} else if (specified_device_name.has_type) { } else if (specified_device_name.has_type) {
return errors::InvalidArgument( return errors::InvalidArgument(
"Could not satisfy explicit device specification '", "Could not satisfy explicit device specification '",
@ -741,7 +740,7 @@ Status SimplePlacer::Run() {
status = colocation_graph.GetDevicesForNode(node, &devices); status = colocation_graph.GetDevicesForNode(node, &devices);
if (!status.ok()) { if (!status.ok()) {
return AttachDef( return AttachDef(
errors::InvalidArgument("Cannot assign a device to node '", errors::InvalidArgument("Cannot assign a device for operation '",
node->name(), "': ", status.error_message()), node->name(), "': ", status.error_message()),
node->def()); node->def());
} }
@ -783,7 +782,7 @@ Status SimplePlacer::Run() {
status = colocation_graph.GetDevicesForNode(node, &devices); status = colocation_graph.GetDevicesForNode(node, &devices);
if (!status.ok()) { if (!status.ok()) {
return AttachDef( return AttachDef(
errors::InvalidArgument("Cannot assign a device to node '", errors::InvalidArgument("Cannot assign a device for operation '",
node->name(), "': ", status.error_message()), node->name(), "': ", status.error_message()),
node->def()); node->def());
} }

29
tensorflow/core/common_runtime/simple_placer_test.cc
View File

@ -939,10 +939,7 @@ TEST_F(SimplePlacerTest, TestUnknownDevice) {
Status s = Place(&g); Status s = Place(&g);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
EXPECT_TRUE( EXPECT_TRUE(StringPiece(s.error_message()).contains("/job:foo"));
StringPiece(s.error_message())
.contains(
"Could not satisfy explicit device specification '/job:foo'"));
} }
// Test that placement fails when the combination of partial // Test that placement fails when the combination of partial
@ -957,10 +954,7 @@ TEST_F(SimplePlacerTest, TestUnknownMergedDevice) {
Status s = Place(&g); Status s = Place(&g);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
EXPECT_TRUE( EXPECT_TRUE(StringPiece(s.error_message()).contains("/job:foo"));
StringPiece(s.error_message())
.contains(
"Could not satisfy explicit device specification '/job:foo'"));
} }
// Test that placement fails when the previously-assigned device for a // Test that placement fails when the previously-assigned device for a
@ -1107,10 +1101,7 @@ TEST_F(SimplePlacerTest, TestNonexistentGpuNoAllowSoftPlacement) {
SessionOptions options; SessionOptions options;
Status s = Place(&g, &options); Status s = Place(&g, &options);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
EXPECT_TRUE(StringPiece(s.error_message()) EXPECT_TRUE(StringPiece(s.error_message()).contains("/device:fakegpu:11"));
.contains("Could not satisfy explicit "
"device specification "
"'/device:fakegpu:11'"));
} }
// Test that placement fails when a node requests an explicit device that is not // Test that placement fails when a node requests an explicit device that is not
@ -1127,10 +1118,7 @@ TEST_F(SimplePlacerTest, TestUnsupportedDeviceNoAllowSoftPlacement) {
SessionOptions options; SessionOptions options;
Status s = Place(&g, &options); Status s = Place(&g, &options);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
EXPECT_TRUE(StringPiece(s.error_message()) EXPECT_TRUE(StringPiece(s.error_message()).contains("/device:fakecpu:0"));
.contains("Could not satisfy explicit "
"device specification "
"'/device:fakecpu:0'"));
EXPECT_TRUE( EXPECT_TRUE(
StringPiece(s.error_message()) StringPiece(s.error_message())
.contains("no supported kernel for fakecpu devices is available")); .contains("no supported kernel for fakecpu devices is available"));
@ -1151,12 +1139,9 @@ TEST_F(SimplePlacerTest, TestNonExistentDevice) {
Status s = Place(&g, &options); Status s = Place(&g, &options);
EXPECT_EQ(error::INVALID_ARGUMENT, s.code()); EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
LOG(WARNING) << s.error_message(); LOG(WARNING) << s.error_message();
EXPECT_TRUE( EXPECT_TRUE(StringPiece(s.error_message())
StringPiece(s.error_message()) .contains("was explicitly assigned to /job:foo/replica:17 "
.contains("Could not satisfy explicit device specification " "but available devices"));
"'/job:foo/replica:17' "
"because no devices matching that specification are "
"registered in this process"));
} }
TEST_F(SimplePlacerTest, TestUnsupportedDeviceAllowSoftPlacement) { TEST_F(SimplePlacerTest, TestUnsupportedDeviceAllowSoftPlacement) {

3
tensorflow/python/framework/test_util_test.py
View File

@ -252,8 +252,7 @@ class TestUtilTest(test_util.TensorFlowTestCase):
self.assertArrayNear(a, b, 0.001) self.assertArrayNear(a, b, 0.001)
def testForceGPU(self): def testForceGPU(self):
with self.assertRaisesRegexp(errors.InvalidArgumentError, with self.assertRaises(errors.InvalidArgumentError):
"Cannot assign a device to node"):
with self.test_session(force_gpu=True): with self.test_session(force_gpu=True):
# this relies on us not having a GPU implementation for assert, which # this relies on us not having a GPU implementation for assert, which
# seems sensible # seems sensible

4
tensorflow/python/training/saver_test.py
View File

@ -1845,8 +1845,8 @@ class MetaGraphTest(test.TestCase):
with session.Session(graph=ops_lib.Graph()) as sess: with session.Session(graph=ops_lib.Graph()) as sess:
saver_module.import_meta_graph( saver_module.import_meta_graph(
meta_graph_def, clear_devices=False, import_scope="new_model") meta_graph_def, clear_devices=False, import_scope="new_model")
with self.assertRaisesRegexp(errors_impl.InvalidArgumentError, # Device refers to GPU, which is not available here.
"Cannot assign a device to node"): with self.assertRaises(errors_impl.InvalidArgumentError):
sess.run(variables.global_variables_initializer()) sess.run(variables.global_variables_initializer())
with session.Session(graph=ops_lib.Graph()) as sess: with session.Session(graph=ops_lib.Graph()) as sess: