2126 lines
81 KiB
Python
2126 lines
81 KiB
Python
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
"""Tests for Grappler LayoutOptimizer."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import numpy as np
|
|
|
|
from tensorflow.core.protobuf import config_pb2
|
|
from tensorflow.core.protobuf import device_properties_pb2
|
|
from tensorflow.core.protobuf import rewriter_config_pb2
|
|
from tensorflow.core.protobuf import saver_pb2
|
|
from tensorflow.python.client import session
|
|
from tensorflow.python.framework import constant_op
|
|
from tensorflow.python.framework import dtypes
|
|
from tensorflow.python.framework import ops
|
|
from tensorflow.python.framework import random_seed
|
|
from tensorflow.python.framework import test_util
|
|
from tensorflow.python.grappler import cluster as gcluster
|
|
from tensorflow.python.grappler import tf_optimizer
|
|
from tensorflow.python.layers import convolutional as conv_layers
|
|
from tensorflow.python.ops import array_ops
|
|
from tensorflow.python.ops import gen_array_ops
|
|
from tensorflow.python.ops import gen_math_ops
|
|
from tensorflow.python.ops import gen_nn_ops
|
|
from tensorflow.python.ops import map_fn
|
|
from tensorflow.python.ops import math_ops
|
|
from tensorflow.python.ops import nn
|
|
from tensorflow.python.ops import random_ops
|
|
from tensorflow.python.ops import state_ops
|
|
from tensorflow.python.ops import variables
|
|
from tensorflow.python.platform import test
|
|
from tensorflow.python.training import gradient_descent
|
|
from tensorflow.python.training import saver as saver_lib
|
|
|
|
|
|
def _weight(shape):
|
|
"""Generates a weight of a given shape."""
|
|
return random_ops.truncated_normal(shape, seed=0, stddev=0.1)
|
|
|
|
|
|
def _bias(shape):
|
|
"""Generates a bias of a given shape."""
|
|
return constant_op.constant(0.1, shape=shape)
|
|
|
|
|
|
def _conv2d(x, w):
|
|
"""Returns a 2d convolution layer with full stride."""
|
|
return nn.conv2d(x, w, strides=[1, 1, 1, 1], padding='SAME')
|
|
|
|
|
|
def _max_pool_2x2(x):
|
|
"""Downsamples a feature map by 2X."""
|
|
return nn.max_pool(
|
|
x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
|
|
|
|
|
|
# Taken from tensorflow/examples/tutorials/mnist/mnist_deep.py
|
|
def _two_layer_model(x):
|
|
x_image = array_ops.reshape(x, [-1, 28, 28, 1])
|
|
w_conv1 = _weight([5, 5, 1, 32])
|
|
b_conv1 = _bias([32])
|
|
h_conv1 = nn.relu(_conv2d(x_image, w_conv1) + b_conv1)
|
|
h_pool1 = _max_pool_2x2(h_conv1)
|
|
w_conv2 = _weight([5, 5, 32, 64])
|
|
b_conv2 = _bias([64])
|
|
h_conv2 = nn.relu(_conv2d(h_pool1, w_conv2) + b_conv2)
|
|
h_pool2 = _max_pool_2x2(h_conv2)
|
|
return h_pool2
|
|
|
|
|
|
def _model_with_second_port():
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([2, 5, 5, 4], seed=0)
|
|
scale = constant_op.constant(0.1, shape=[4])
|
|
offset = constant_op.constant(0.3, shape=[4])
|
|
y, mean, _ = nn.fused_batch_norm(x, scale, offset)
|
|
mul = math_ops.add(y, mean)
|
|
output = array_ops.identity(mul)
|
|
return output
|
|
|
|
|
|
def _model_with_branch(x):
|
|
x_image = array_ops.reshape(x, [-1, 28, 28, 1])
|
|
w_conv1 = _weight([5, 5, 1, 32])
|
|
w_conv2 = _weight([5, 5, 1, 32])
|
|
c_conv1 = _conv2d(x_image, w_conv1)
|
|
c_conv2 = _conv2d(x_image, w_conv2)
|
|
add = math_ops.add(c_conv1, c_conv2)
|
|
return add
|
|
|
|
|
|
def _model_with_vec_and_4d(x):
|
|
x_image = array_ops.reshape(x, [-1, 28, 28, 1])
|
|
w_conv1 = _weight([5, 5, 1, 32])
|
|
c_conv1 = _conv2d(x_image, w_conv1)
|
|
vector = constant_op.constant(6.4, shape=[32])
|
|
add = math_ops.add(c_conv1, vector)
|
|
return add
|
|
|
|
|
|
def _loop():
|
|
random_seed.set_random_seed(0)
|
|
x1 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x2 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x3 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x4 = random_ops.truncated_normal([1, 784], seed=0)
|
|
elems = (x1, x2, x3, x4)
|
|
outputs = map_fn.map_fn(_two_layer_model, elems, dtype=dtypes.float32)
|
|
return outputs
|
|
|
|
|
|
def _loop_with_branch():
|
|
random_seed.set_random_seed(0)
|
|
x1 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x2 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x3 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x4 = random_ops.truncated_normal([1, 784], seed=0)
|
|
elems = (x1, x2, x3, x4)
|
|
outputs = map_fn.map_fn(_model_with_branch, elems, dtype=dtypes.float32)
|
|
return outputs
|
|
|
|
|
|
def _loop_with_vec_and_4d():
|
|
random_seed.set_random_seed(0)
|
|
x1 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x2 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x3 = random_ops.truncated_normal([1, 784], seed=0)
|
|
x4 = random_ops.truncated_normal([1, 784], seed=0)
|
|
elems = (x1, x2, x3, x4)
|
|
outputs = map_fn.map_fn(_model_with_vec_and_4d, elems, dtype=dtypes.float32)
|
|
return outputs
|
|
|
|
|
|
def _get_config(layout_optimizer=True):
|
|
if layout_optimizer:
|
|
rewrite_options = rewriter_config_pb2.RewriterConfig(
|
|
layout_optimizer=rewriter_config_pb2.RewriterConfig.ON,
|
|
# do not remove duplicated nodes
|
|
arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF)
|
|
else:
|
|
rewrite_options = rewriter_config_pb2.RewriterConfig(
|
|
layout_optimizer=rewriter_config_pb2.RewriterConfig.OFF,
|
|
# do not remove duplicated nodes
|
|
arithmetic_optimization=rewriter_config_pb2.RewriterConfig.OFF)
|
|
rewrite_options.min_graph_nodes = -1
|
|
graph_options = config_pb2.GraphOptions(
|
|
rewrite_options=rewrite_options, build_cost_model=1)
|
|
config = config_pb2.ConfigProto(graph_options=graph_options)
|
|
config.graph_options.optimizer_options.opt_level = -1
|
|
return config
|
|
|
|
|
|
def _simple_metagraph(depthwise=False):
|
|
random_seed.set_random_seed(0)
|
|
x = variables.Variable(random_ops.truncated_normal([1, 200, 200, 3], seed=0))
|
|
conv = conv_layers.separable_conv2d if depthwise else conv_layers.conv2d
|
|
y = conv(x, 32, [3, 3])
|
|
z = conv(y, 32, [3, 3])
|
|
optimizer = gradient_descent.GradientDescentOptimizer(1e-4)
|
|
loss = math_ops.reduce_mean(z)
|
|
train_op = optimizer.minimize(loss)
|
|
graph = ops.get_default_graph()
|
|
graph.add_to_collection('train_op', train_op)
|
|
meta_graph = saver_lib.export_meta_graph(graph_def=graph.as_graph_def())
|
|
return meta_graph
|
|
|
|
|
|
def _get_cluster():
|
|
named_device = device_properties_pb2.NamedDevice()
|
|
named_device.name = '/GPU:0'
|
|
named_device.properties.type = 'GPU'
|
|
named_device.properties.num_cores = 24
|
|
named_device.properties.frequency = 1000
|
|
named_device.properties.environment['architecture'] = '4'
|
|
cluster = gcluster.Cluster(devices=[named_device])
|
|
return cluster
|
|
|
|
|
|
def _is_transpose(node):
|
|
return node.endswith('TransposeNHWCToNCHW-LayoutOptimizer') or node.endswith(
|
|
'TransposeNCHWToNHWC-LayoutOptimizer') or node.endswith(
|
|
'TransposeNDHWCToNCDHW-LayoutOptimizer') or node.endswith(
|
|
'TransposeNCDHWToNDHWC-LayoutOptimizer')
|
|
|
|
|
|
def _is_permute(node):
|
|
return node.endswith('VecPermuteNHWCToNCHW-LayoutOptimizer') or node.endswith(
|
|
'VecPermuteNCHWToNHWC-LayoutOptimizer')
|
|
|
|
|
|
@test_util.for_all_test_methods(test_util.no_xla_auto_jit,
|
|
'Test does not apply in XLA setting')
|
|
class LayoutOptimizerTest(test.TestCase):
|
|
"""Tests the Grappler layout optimizer."""
|
|
|
|
def _assert_trans_nchw_to_nhwc(self, name, nodes):
|
|
self.assertIn(name + '-TransposeNCHWToNHWC-LayoutOptimizer', nodes)
|
|
|
|
def _assert_trans_nhwc_to_nchw(self, name, nodes):
|
|
self.assertIn(name + '-TransposeNHWCToNCHW-LayoutOptimizer', nodes)
|
|
|
|
def _assert_trans_ncdhw_to_ndhwc(self, name, nodes):
|
|
self.assertIn(name + '-TransposeNCDHWToNDHWC-LayoutOptimizer', nodes)
|
|
|
|
def _assert_trans_ndhwc_to_ncdhw(self, name, nodes):
|
|
self.assertIn(name + '-TransposeNDHWCToNCDHW-LayoutOptimizer', nodes)
|
|
|
|
def _assert_map_nhwc_to_nchw(self, name, nodes):
|
|
self.assertIn(name + '-DimMapNHWCToNCHW-LayoutOptimizer', nodes)
|
|
|
|
def _assert_map_ndhwc_to_ncdhw(self, name, nodes):
|
|
self.assertIn(name + '-DataFormatDimMapNDHWCToNCDHW-LayoutOptimizer', nodes)
|
|
|
|
def _assert_vec_nchw_to_nhwc(self, name, nodes):
|
|
self.assertIn(name + '-VecPermuteNCHWToNHWC-LayoutOptimizer', nodes)
|
|
|
|
def _assert_vec_nhwc_to_nchw(self, name, nodes):
|
|
self.assertIn(name + '-VecPermuteNHWCToNCHW-LayoutOptimizer', nodes)
|
|
|
|
def _assert_vec_ncdhw_to_ndhwc(self, name, nodes):
|
|
self.assertIn(name + '-DataFormatVecPermuteNCDHWToNDHWC-LayoutOptimizer',
|
|
nodes)
|
|
|
|
def _assert_vec_ndhwc_to_ncdhw(self, name, nodes):
|
|
self.assertIn(name + '-DataFormatVecPermuteNDHWCToNCDHW-LayoutOptimizer',
|
|
nodes)
|
|
|
|
def _train(self, checkpoint_path, layout_optimizer=False, restore=False):
|
|
ops.reset_default_graph()
|
|
graph = ops.get_default_graph()
|
|
with session.Session(
|
|
config=_get_config(layout_optimizer), graph=graph) as sess:
|
|
batch = 2
|
|
height = 6
|
|
width = 7
|
|
input_channels = 3
|
|
shape = [batch, height, width, input_channels]
|
|
image = array_ops.placeholder(dtype='float32', shape=shape)
|
|
conv1 = conv_layers.conv2d(image, 32, [3, 3])
|
|
conv2 = conv_layers.conv2d(conv1, 32, [3, 3])
|
|
optimizer = gradient_descent.GradientDescentOptimizer(0.01)
|
|
loss = math_ops.reduce_mean(conv2)
|
|
train_op = optimizer.minimize(loss)
|
|
saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
|
|
|
|
if restore:
|
|
saver.restore(sess, checkpoint_path)
|
|
else:
|
|
self.evaluate(variables.global_variables_initializer())
|
|
|
|
np.random.seed(0)
|
|
for _ in range(2):
|
|
image_val = np.random.rand(*shape).astype(np.float32)
|
|
sess.run([loss, train_op], feed_dict={image: image_val})
|
|
|
|
if restore:
|
|
all_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
|
|
all_vars_values = [var.eval(session=sess) for var in all_vars]
|
|
return all_vars_values
|
|
else:
|
|
saver.save(sess, checkpoint_path)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testTwoConvLayers(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
output = _two_layer_model(x)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Relu_1-0-0', nodes)
|
|
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSplitWithNonConstAxis(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
dim = array_ops.placeholder(dtype='int32')
|
|
split = array_ops.split(conv, 2, axis=dim)
|
|
scale = constant_op.constant(0.1, shape=[32])
|
|
offset = constant_op.constant(0.3, shape=[32])
|
|
bn0 = nn.fused_batch_norm(split[0], scale, offset)
|
|
bn1 = nn.fused_batch_norm(split[1], scale, offset)
|
|
add = bn0[0] + bn1[0]
|
|
output = array_ops.identity(add)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={dim: 3})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata, feed_dict={dim: 3})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('add_2-0-0', nodes)
|
|
self._assert_map_nhwc_to_nchw('split-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSplitVWithNonConstAxis(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
dim = array_ops.placeholder(dtype='int32')
|
|
sizes = constant_op.constant([50, 10, 4], shape=[3])
|
|
split = gen_array_ops.split_v(
|
|
value=conv, size_splits=sizes, axis=dim, num_split=3)
|
|
output = math_ops.reduce_sum(split[0])
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={dim: 3})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata, feed_dict={dim: 3})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('SplitV-0-0', nodes)
|
|
self._assert_map_nhwc_to_nchw('SplitV-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testPadWithConstPaddings(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
paddings_val = [[1, 2], [3, 4], [5, 6], [7, 8]]
|
|
paddings = constant_op.constant(
|
|
paddings_val, dtype='int32', name='PaddingsConst')
|
|
pad = array_ops.pad(conv, paddings)
|
|
output = array_ops.identity(pad)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Pad-0-0', nodes)
|
|
self.assertIn('Pad-1-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReduceSum(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv)
|
|
output = array_ops.identity(reduce_sum)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Three transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testCast(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
cast = math_ops.cast(conv, dtype='bool')
|
|
output = array_ops.identity(cast)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Cast-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSqueeze(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[1, 2])
|
|
squeeze = array_ops.squeeze(reduce_sum)
|
|
output = array_ops.identity(squeeze)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Three transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSqueezeAlongHW(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[1, 2], keepdims=True)
|
|
squeeze = array_ops.squeeze(reduce_sum, axis=[1, 2])
|
|
output = array_ops.identity(squeeze)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Three transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSqueezeAlongNHW(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[0, 1, 2], keepdims=True)
|
|
squeeze = array_ops.squeeze(reduce_sum, axis=[0, 1, 2])
|
|
output = array_ops.identity(squeeze)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Three transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReduceSumAlongHWC(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[1, 2, 3])
|
|
output = array_ops.identity(reduce_sum)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Three transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReduceSumAlongNHW(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[0, 1, 2])
|
|
output = array_ops.identity(reduce_sum)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Three transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReduceSumAlongC(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[3])
|
|
output = array_ops.identity(reduce_sum)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Three transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReduceSumAlongCKeepDims(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[3], keepdims=True)
|
|
output = array_ops.identity(reduce_sum)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Sum-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReduceSumAlongHKeepDims(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[2], keepdims=True)
|
|
output = array_ops.identity(reduce_sum)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReduceSumAlongWCKeepDims(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
reduce_sum = math_ops.reduce_sum(conv, axis=[2, 3], keepdims=True)
|
|
output = array_ops.identity(reduce_sum)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testConcatWithControlDependency(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
axis = constant_op.constant(3)
|
|
var = variables.Variable(3)
|
|
assign = state_ops.assign(var, 6)
|
|
with ops.control_dependencies([assign]):
|
|
concat = array_ops.concat([conv, conv], axis)
|
|
output = array_ops.identity(concat)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('concat-0-0', nodes)
|
|
self.assertIn('concat-2-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testConcatWithControlDependencyFor5DTensor(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([2, 2, 14, 14, 1], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 1, 2], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
y = gen_nn_ops.conv3d(x, w, strides, 'SAME')
|
|
axis = constant_op.constant(4)
|
|
var = variables.Variable(3)
|
|
assign = state_ops.assign(var, 6)
|
|
with ops.control_dependencies([assign]):
|
|
concat = array_ops.concat([y, y], axis)
|
|
output = array_ops.identity(concat)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('concat-0-0', nodes)
|
|
self._assert_map_ndhwc_to_ncdhw('concat-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testFill(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = array_ops.placeholder(dtype='float32')
|
|
conv = _two_layer_model(x)
|
|
shape = array_ops.shape(conv)
|
|
scalar = array_ops.constant(5.7)
|
|
fill = array_ops.fill(shape, scalar)
|
|
output = array_ops.identity(fill)
|
|
|
|
x_val = [3.4] * 784
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={x: x_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
x: x_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
num_vec_permute = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
if _is_permute(node.name):
|
|
num_vec_permute += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
# Two vector permute nodes were initially added in the Expand phase of
|
|
# LayoutOptimizer; they cancelled out each other in the Collapse phase.
|
|
expected_vec_permute = 0
|
|
self.assertEqual(expected_vec_permute, num_vec_permute)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Fill-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testTile(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
multiple = array_ops.placeholder(dtype='int32')
|
|
tile = array_ops.tile(conv, multiple)
|
|
output = array_ops.identity(tile)
|
|
|
|
multiple_val = [2, 3, 4, 1]
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={multiple: multiple_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
multiple: multiple_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Tile-0-0', nodes)
|
|
self._assert_vec_nhwc_to_nchw('Tile-1', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReverseWithConstDims(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
dims = constant_op.constant([3, 1], name='DimsConst')
|
|
reverse = array_ops.reverse(conv, dims)
|
|
output = array_ops.identity(reverse)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('ReverseV2-0-0', nodes)
|
|
self.assertIn('ReverseV2-1-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReverseWithNonConstDims(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
dims = array_ops.placeholder(dtype='int32')
|
|
reverse = array_ops.reverse(conv, dims)
|
|
output = array_ops.identity(reverse)
|
|
|
|
dims_val = [2, 3]
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={dims: dims_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
dims: dims_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('ReverseV2-0-0', nodes)
|
|
self._assert_map_nhwc_to_nchw('ReverseV2-1', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSelectOp(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
add = math_ops.add(conv, conv)
|
|
mean = math_ops.reduce_mean(conv)
|
|
condition = math_ops.less(conv, mean)
|
|
select = gen_math_ops.select(condition, conv, add)
|
|
output = array_ops.identity(select)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Select-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSelectOpConditionUnknownShape(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
add = math_ops.add(conv, conv)
|
|
condition = array_ops.placeholder(dtype='bool')
|
|
select = gen_math_ops.select(condition, conv, add)
|
|
output = array_ops.identity(select)
|
|
|
|
condition_val = np.zeros((1, 7, 7, 64))
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={condition: condition_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={condition: condition_val})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 3
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSelectOpScalarCondition(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
add = math_ops.add(conv, conv)
|
|
condition = constant_op.constant(True)
|
|
select = gen_math_ops.select(condition, conv, add)
|
|
output = array_ops.identity(select)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Select-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testPadWithNonConstPaddings(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
paddings = array_ops.placeholder(dtype='int32')
|
|
pad = array_ops.pad(conv, paddings)
|
|
output = array_ops.identity(pad)
|
|
|
|
paddings_val = [[1, 2], [3, 4], [5, 6], [7, 8]]
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={paddings: paddings_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
paddings: paddings_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Pad-0-0', nodes)
|
|
self._assert_vec_nhwc_to_nchw('Pad-1', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testMaxPoolV2(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
ksize = constant_op.constant([1, 2, 3, 1], shape=[4])
|
|
strides = array_ops.placeholder(dtype='int32', shape=[4])
|
|
max_pool = gen_nn_ops.max_pool_v2(conv, ksize, strides, 'VALID')
|
|
output = array_ops.identity(max_pool)
|
|
|
|
strides_val = [1, 3, 2, 1]
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={strides: strides_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
strides: strides_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('MaxPoolV2-0-0', nodes)
|
|
self._assert_vec_nhwc_to_nchw('MaxPoolV2-2', nodes)
|
|
self.assertIn('MaxPoolV2-1-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testMaxPoolGradV2(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
ksize = constant_op.constant([1, 2, 3, 1], shape=[4])
|
|
strides = array_ops.placeholder(dtype='int32', shape=[4])
|
|
max_pool_grad = gen_nn_ops.max_pool_grad_v2(conv, conv, conv, ksize,
|
|
strides, 'VALID')
|
|
output = array_ops.identity(max_pool_grad)
|
|
|
|
strides_val = [1, 3, 2, 1]
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={strides: strides_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
strides: strides_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('MaxPoolGradV2-0-0', nodes)
|
|
self._assert_vec_nhwc_to_nchw('MaxPoolGradV2-4', nodes)
|
|
self.assertIn('MaxPoolGradV2-3-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testLeakyRelu(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([4, 14, 14, 1], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 1, 2], seed=0)
|
|
y = nn.conv2d(x, w, strides=[1, 1, 1, 1], padding='SAME')
|
|
y = nn.leaky_relu(y, alpha=0.2)
|
|
output = array_ops.identity(y)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nchw_to_nhwc('LeakyRelu-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testLeakyReluGrad(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([4, 14, 14, 1], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 1, 1], seed=0)
|
|
y = nn.conv2d(x, w, strides=[1, 1, 1, 1], padding='SAME')
|
|
y = gen_nn_ops.leaky_relu_grad(y, x, alpha=0.2)
|
|
output = array_ops.identity(y)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 3
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('LeakyReluGrad-1', nodes)
|
|
self._assert_trans_nchw_to_nhwc('LeakyReluGrad-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testLeakyReluGradFor5DTensors(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 4, 2, 3, 3], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 3, 3], seed=0)
|
|
y = gen_nn_ops.conv3d(x, w, [1, 1, 1, 1, 1], 'SAME')
|
|
y = gen_nn_ops.leaky_relu_grad(y, x, alpha=0.2)
|
|
output = array_ops.identity(y)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 3
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('LeakyReluGrad-1', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('LeakyReluGrad-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testReduceOpsFor5DTensors(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 4, 2, 3, 3], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 3, 3], seed=0)
|
|
conv3d = gen_nn_ops.conv3d(x, w, [1, 1, 1, 1, 1], 'SAME')
|
|
y = math_ops.reduce_mean(conv3d, [0, 1, 2, 3], keepdims=True)
|
|
output = array_ops.identity(y)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# The reduce op Mean needs to dim map the input reduce index to NCDHW.
|
|
# Then, the output needs to be tranposed back to NDHWC.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_map_ndhwc_to_ncdhw('Mean-1', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('Mean-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testBinaryOpsFor5DTensors(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 4, 2, 3, 3], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 3, 3], seed=0)
|
|
mean = random_ops.truncated_normal([1, 1, 1, 1, 3], seed=0)
|
|
variance = random_ops.truncated_normal([1, 1, 1, 1, 3], seed=0)
|
|
gamma = random_ops.truncated_normal([1, 1, 1, 1, 3], seed=0)
|
|
beta = random_ops.truncated_normal([1, 1, 1, 1, 3], seed=0)
|
|
conv3d = gen_nn_ops.conv3d(x, w, [1, 1, 1, 1, 1], 'SAME')
|
|
y = nn.batch_normalization(
|
|
conv3d,
|
|
mean=mean,
|
|
variance=variance,
|
|
scale=gamma,
|
|
offset=beta,
|
|
variance_epsilon=0.001)
|
|
output = array_ops.identity(y)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# The binary ops mul_1 and add_1 in batch norm need to transpose one of
|
|
# the two inputs to NCDHW. The other input has already been tranposed via
|
|
# Conv3D.
|
|
expected_num_transposes = 4
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_trans_ndhwc_to_ncdhw('batchnorm/mul_1-1', nodes)
|
|
self._assert_trans_ndhwc_to_ncdhw('batchnorm/add_1-1', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('batchnorm/add_1-0-0', nodes)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testBatchNorm3D(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x_3d = random_ops.truncated_normal([1, 4, 2, 3, 3], seed=0)
|
|
filters = random_ops.truncated_normal([2, 2, 2, 3, 3], seed=0)
|
|
strides_val = [1, 1, 1, 1, 1]
|
|
scale = constant_op.constant(0.1, shape=[3])
|
|
offset = constant_op.constant(0.3, shape=[3])
|
|
conv3d = gen_nn_ops.conv3d(x_3d, filters, strides_val, 'SAME')
|
|
y, _, _ = nn.fused_batch_norm(conv3d, scale, offset, data_format='NDHWC')
|
|
output = array_ops.identity(y)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('FusedBatchNormV3-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testBatchNormGrad3D(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x_3d = random_ops.truncated_normal([1, 4, 2, 3, 3], seed=0)
|
|
filters = random_ops.truncated_normal([2, 2, 2, 3, 3], seed=0)
|
|
strides_val = [1, 1, 1, 1, 1]
|
|
scale = constant_op.constant(0.1, shape=[3])
|
|
offset = constant_op.constant(0.3, shape=[3])
|
|
mean = constant_op.constant(0.1, shape=[3])
|
|
variance = constant_op.constant(0.3, shape=[3])
|
|
conv3d = gen_nn_ops.conv3d(x_3d, filters, strides_val, 'SAME')
|
|
y, running_mean, running_var, r0, r1, r2 = gen_nn_ops.fused_batch_norm_v3(
|
|
conv3d,
|
|
scale,
|
|
offset,
|
|
mean,
|
|
variance,
|
|
epsilon=1.001e-5,
|
|
exponential_avg_factor=1.0,
|
|
data_format='NDHWC',
|
|
is_training=True,
|
|
name='batch_norm')
|
|
dx, dscale, doffset, _, _ = gen_nn_ops.fused_batch_norm_grad_v3(
|
|
y,
|
|
x_3d,
|
|
scale,
|
|
r0,
|
|
r1,
|
|
r2,
|
|
epsilon=1.001e-5,
|
|
data_format='NDHWC',
|
|
is_training=True)
|
|
output = array_ops.identity(dx)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 3
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_trans_ndhwc_to_ncdhw('FusedBatchNormGradV3-1', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('FusedBatchNormGradV3-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testConv3D(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([2, 2, 14, 14, 1], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 1, 2], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
y = gen_nn_ops.conv3d(x, w, strides, 'SAME')
|
|
output = array_ops.identity(y)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('Conv3D-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testConv3DBackpropInput(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
random_seed.set_random_seed(0)
|
|
dy = random_ops.truncated_normal([2, 2, 14, 14, 1], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 1, 1], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
x_shape = array_ops.shape(dy)
|
|
dx = gen_nn_ops.conv3d_backprop_input_v2(x_shape, w, dy, strides, 'SAME')
|
|
output = array_ops.identity(dx)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_vec_ndhwc_to_ncdhw('Conv3DBackpropInputV2-0', nodes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3DBackpropInputV2-2', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('Conv3DBackpropInputV2-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testConv3DBackpropFilter(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([2, 2, 14, 14, 1], seed=0)
|
|
dy = random_ops.truncated_normal([2, 2, 14, 14, 1], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
w_shape = constant_op.constant([2, 2, 2, 1, 1], shape=[5])
|
|
dw = gen_nn_ops.conv3d_backprop_filter_v2(x, w_shape, dy, strides, 'SAME')
|
|
output = array_ops.identity(dw)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3DBackpropFilterV2-0', nodes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3DBackpropFilterV2-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testBiasAddFor5DTensor(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([2, 2, 14, 14, 1], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 1, 2], seed=0)
|
|
b = random_ops.truncated_normal([2], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
y = gen_nn_ops.conv3d(x, w, strides, 'SAME')
|
|
y = gen_nn_ops.bias_add(y, b, 'NHWC')
|
|
output = array_ops.identity(y)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('BiasAdd-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testBiasAddGradFor5DTensor(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
random_seed.set_random_seed(0)
|
|
dy = random_ops.truncated_normal([2, 2, 14, 14, 1], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 1, 1], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
dy_shape = array_ops.shape(dy)
|
|
dx = gen_nn_ops.conv3d_backprop_input_v2(dy_shape, w, dy, strides, 'SAME')
|
|
db = gen_nn_ops.bias_add_grad(dx, 'NHWC')
|
|
output = array_ops.identity(db)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# The output of Conv3DBackpropInputV2 won't be converted back to NDHWC
|
|
# because of the BiasAddGrad.
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_vec_ndhwc_to_ncdhw('Conv3DBackpropInputV2-0', nodes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3DBackpropInputV2-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSliceWithNonConstAxis(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
size = array_ops.placeholder(dtype='int32')
|
|
s = array_ops.slice(conv, [0, 0, 0, 0], size)
|
|
output = array_ops.identity(s)
|
|
|
|
size_val = [1, 2, 3, 4]
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={size: size_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
size: size_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Slice-0-0', nodes)
|
|
self._assert_vec_nhwc_to_nchw('Slice-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testSliceWithNonConstAxisFor5DTensor(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([2, 2, 14, 14, 1], seed=0)
|
|
w = random_ops.truncated_normal([2, 2, 2, 1, 2], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
y = gen_nn_ops.conv3d(x, w, strides, 'SAME')
|
|
size = array_ops.placeholder(dtype='int32')
|
|
s = array_ops.slice(y, [0, 0, 0, 0, 0], size)
|
|
output = array_ops.identity(s)
|
|
|
|
size_val = [1, 1, 2, 2, 1]
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={size: size_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={size: size_val})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('Slice-0-0', nodes)
|
|
self._assert_vec_ndhwc_to_ncdhw('Slice-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testStridedSliceWithNonConstAxis(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
end = array_ops.placeholder(dtype='int32')
|
|
s = array_ops.strided_slice(conv, [0, 0, 0, 0], end, strides=[1, 2, 3, 1])
|
|
output = array_ops.identity(s)
|
|
|
|
end_val = [1, 2, 3, 4]
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={end: end_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
end: end_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('StridedSlice-0-0', nodes)
|
|
self._assert_vec_nhwc_to_nchw('StridedSlice-2', nodes)
|
|
self.assertIn('StridedSlice-1-LayoutOptimizer', nodes)
|
|
self.assertIn('StridedSlice-3-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testStridedSliceWithMask1011(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
# This will generate a StridedSlice op with begin mask and
|
|
# end mask 11(1011).
|
|
s = conv[:, :, 1:-1, :]
|
|
output = array_ops.identity(s)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('strided_slice-0-0', nodes)
|
|
self.assertIn('strided_slice-1-LayoutOptimizer', nodes)
|
|
self.assertIn('strided_slice-2-LayoutOptimizer', nodes)
|
|
self.assertIn('strided_slice-3-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testStridedSliceWithMask0111(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
# This will generate a StridedSlice op with begin mask and
|
|
# end mask 7(0111).
|
|
s = conv[:, :, :, 1:-1]
|
|
output = array_ops.identity(s)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('strided_slice-0-0', nodes)
|
|
self.assertIn('strided_slice-1-LayoutOptimizer', nodes)
|
|
self.assertIn('strided_slice-2-LayoutOptimizer', nodes)
|
|
self.assertIn('strided_slice-3-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testStridedSliceGradWithNonConstAxis(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
random_seed.set_random_seed(0)
|
|
x = random_ops.truncated_normal([1, 784], seed=0)
|
|
conv = _two_layer_model(x)
|
|
end = array_ops.placeholder(dtype='int32')
|
|
shape = array_ops.shape(conv)
|
|
end_val = [1, 2, 3, 4]
|
|
s = array_ops.strided_slice(
|
|
conv, [0, 0, 0, 0], end_val, strides=[1, 2, 3, 1])
|
|
s_grad = array_ops.strided_slice_grad(shape, [0, 0, 0, 0], end,
|
|
[1, 2, 3, 1], s)
|
|
output = array_ops.identity(s_grad)
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={end: end_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
end: end_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('StridedSliceGrad-0-0', nodes)
|
|
self._assert_vec_nhwc_to_nchw('StridedSliceGrad-2', nodes)
|
|
self.assertIn('StridedSlice-1-LayoutOptimizer', nodes)
|
|
self.assertIn('StridedSlice-2-LayoutOptimizer', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testShapeN(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
x = array_ops.placeholder(dtype='float32')
|
|
conv = _two_layer_model(x)
|
|
shapen = array_ops.shape_n([conv, conv])
|
|
output = math_ops.add(shapen[0], shapen[1])
|
|
|
|
x_val = [1.7] * 784
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={x: x_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={
|
|
x: x_val
|
|
})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_vec_nchw_to_nhwc('ShapeN-0-0', nodes)
|
|
self.assertAllEqual(output_val_ref, output_val)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testShapeNFor5DTensor(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
h = array_ops.placeholder(dtype='float32')
|
|
x = array_ops.reshape(h, [-1, 2, 14, 14, 1])
|
|
w = random_ops.truncated_normal([2, 2, 2, 1, 2], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
y = gen_nn_ops.conv3d(x, w, strides, 'SAME')
|
|
shapen = array_ops.shape_n([y, y])
|
|
output = math_ops.add(shapen[0], shapen[1])
|
|
|
|
x_val = [1.7] * 784
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={h: x_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata, feed_dict={h: x_val})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 1
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_vec_ncdhw_to_ndhwc('ShapeN-0-0', nodes)
|
|
self._assert_vec_ncdhw_to_ndhwc('ShapeN-1-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testIdentityNFor4DAnd5DTensors(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
h = array_ops.placeholder(dtype='float32')
|
|
x = array_ops.reshape(h, [-1, 2, 14, 14, 1])
|
|
w = random_ops.truncated_normal([2, 2, 2, 1, 4], seed=0)
|
|
strides = [1, 1, 1, 1, 1]
|
|
y = gen_nn_ops.conv3d(x, w, strides, 'SAME')
|
|
x1 = array_ops.reshape(h, [-1, 784])
|
|
y1 = _two_layer_model(x1)
|
|
outputs = array_ops.identity_n([y1, y])
|
|
new_x0 = array_ops.reshape(outputs[0], [-1, 2, 14, 14, 1])
|
|
new_x1 = array_ops.reshape(outputs[1], [-1, 2, 14, 14, 1])
|
|
output = math_ops.add(new_x0, new_x1)
|
|
|
|
x_val = [1.7] * 784
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={h: x_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata, feed_dict={h: x_val})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 4
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self._assert_trans_ncdhw_to_ndhwc('IdentityN-1-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('IdentityN-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testShapeNFollowedByNotConvertibleNodeReshape(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
x = array_ops.placeholder(dtype='float32')
|
|
conv = _two_layer_model(x)
|
|
conv_reshape = array_ops.reshape(conv, [1, 1, 1, -1])
|
|
shapen = array_ops.shape_n([conv, conv_reshape])
|
|
shape = array_ops.identity(shapen[1])
|
|
ones = array_ops.ones(shape)
|
|
output = math_ops.add_n([conv_reshape, ones])
|
|
|
|
x_val = [1.7] * 784
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = sess.run(output, feed_dict={x: x_val})
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(
|
|
output, run_metadata=metadata, feed_dict={x: x_val})
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testLoop(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
output = _loop()
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
# Four transposes were initially added in the Expand phase of
|
|
# LayoutOptimizer; two of them are cancelled out in the Collapse phase.
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('map/while/Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('map/while/MaxPool_1-0-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testLoopWithBranch(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
output = _loop_with_branch()
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 3
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('map/while/Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('map/while/Add_1-0-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testLoopWithVecAnd4D(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
output = _loop_with_vec_and_4d()
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('map/while/Conv2D-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('map/while/Add_1-0-2', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testBinaryOpSecondPort(self):
|
|
if test.is_gpu_available(cuda_only=True):
|
|
output = _model_with_second_port()
|
|
|
|
with session.Session(config=_get_config(False)) as sess:
|
|
output_val_ref = self.evaluate(output)
|
|
|
|
with session.Session(config=_get_config()) as sess:
|
|
metadata = config_pb2.RunMetadata()
|
|
output_val = sess.run(output, run_metadata=metadata)
|
|
|
|
nodes = []
|
|
num_transposes = 0
|
|
for node in metadata.cost_graph.node:
|
|
if _is_transpose(node.name):
|
|
num_transposes += 1
|
|
nodes.append(node.name)
|
|
|
|
expected_num_transposes = 2
|
|
self.assertEqual(expected_num_transposes, num_transposes)
|
|
self._assert_trans_nhwc_to_nchw('FusedBatchNormV3-0', nodes)
|
|
self._assert_trans_nchw_to_nhwc('Add-0-0', nodes)
|
|
self.assertAllClose(output_val_ref, output_val, atol=1e-3)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testGradient(self):
|
|
meta_graph = _simple_metagraph()
|
|
config = config_pb2.ConfigProto()
|
|
config.graph_options.rewrite_options.CopyFrom(
|
|
rewriter_config_pb2.RewriterConfig(
|
|
layout_optimizer=rewriter_config_pb2.RewriterConfig.ON,
|
|
min_graph_nodes=-1))
|
|
optimized_graph = tf_optimizer.OptimizeGraph(
|
|
config, meta_graph, cluster=_get_cluster())
|
|
|
|
found = 0
|
|
for node in optimized_graph.node:
|
|
if node.op in ['Conv2D', 'Conv2DBackpropFilter', 'Conv2DBackpropInput']:
|
|
found += 1
|
|
self.assertEqual(node.attr['data_format'].s, b'NCHW')
|
|
self.assertEqual(found, 5)
|
|
|
|
@test_util.deprecated_graph_mode_only
|
|
def testDepthwise(self):
|
|
meta_graph = _simple_metagraph(depthwise=True)
|
|
config = config_pb2.ConfigProto()
|
|
config.graph_options.rewrite_options.CopyFrom(
|
|
rewriter_config_pb2.RewriterConfig(
|
|
layout_optimizer=rewriter_config_pb2.RewriterConfig.ON,
|
|
min_graph_nodes=-1))
|
|
optimized_graph = tf_optimizer.OptimizeGraph(
|
|
config, meta_graph, cluster=_get_cluster())
|
|
|
|
found = 0
|
|
for node in optimized_graph.node:
|
|
if node.op in [
|
|
'DepthwiseConv2dNative', 'DepthwiseConv2dNativeBackpropFilter',
|
|
'DepthwiseConv2dNativeBackpropInput'
|
|
]:
|
|
found += 1
|
|
self.assertEqual(node.attr['data_format'].s, b'NCHW')
|
|
self.assertEqual(found, 6)
|
|
|
|
def testCheckpointCompatibility(self):
|
|
if not test.is_gpu_available(cuda_only=True):
|
|
self.skipTest('GPU required')
|
|
|
|
checkpoint_path = self.get_temp_dir()
|
|
self._train(checkpoint_path)
|
|
vars_expected = self._train(checkpoint_path, restore=True)
|
|
vars_layout_optimized = self._train(
|
|
checkpoint_path, restore=True, layout_optimizer=True)
|
|
|
|
for var_expected, var_layout_optimized in zip(vars_expected,
|
|
vars_layout_optimized):
|
|
self.assertAllClose(var_expected, var_layout_optimized, atol=1e-6)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
test.main()
|