From 543f61dcab11cc3461c97dcdce660536e3e498d7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 24 Oct 2019 14:44:53 -0700 Subject: [PATCH] Add quantize tests for activation ops: relu, relu1, relu6 fused with conv, and tanh not fused with conv. PiperOrigin-RevId: 276569841 Change-Id: I3bdda2d46dc75babd61cbebb7fa5dbc6a3abd737 --- tensorflow/lite/build_def.bzl | 3 + tensorflow/lite/testing/BUILD | 28 ++-- .../lite/testing/generate_examples_lib.py | 1 + .../lite/testing/op_tests/conv_activation.py | 139 ++++++++++++++++++ tensorflow/lite/testing/op_tests/tanh.py | 7 +- tensorflow/lite/testing/toco_convert.py | 6 +- 6 files changed, 169 insertions(+), 15 deletions(-) create mode 100644 tensorflow/lite/testing/op_tests/conv_activation.py diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl index 430deddd573..ee0c4d79852 100644 --- a/tensorflow/lite/build_def.bzl +++ b/tensorflow/lite/build_def.bzl @@ -243,6 +243,9 @@ def generated_test_models(): "constant", "control_dep", "conv", + "conv_relu", + "conv_relu1", + "conv_relu6", "conv2d_transpose", "conv_with_shared_weights", "conv_to_depthwiseconv_with_shared_weights", diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD index 41b70ea0889..a9bc36b30df 100644 --- a/tensorflow/lite/testing/BUILD +++ b/tensorflow/lite/testing/BUILD @@ -480,26 +480,30 @@ tf_py_wrap_cc( tflite_portable_test_suite() edgetpu_ops = [ - "conv", # high error - "fully_connected", - "softmax", - "reshape", "add", - "mul", - "sub", "avg_pool", - "max_pool", "concat", - "resize_bilinear", - "l2norm", # high error - "sum", # high error + "conv", # high error + "conv_relu", + "conv_relu1", + "conv_relu6", "depthwiseconv", # high error + "fully_connected", + "l2norm", # high error + "max_pool", + "mul", + "pad", # high error + "reshape", + "resize_bilinear", + "slice", + "softmax", "space_to_depth", "split", "squeeze", - "pad", # high error - "slice", "strided_slice", + "sub", + "sum", # high error + "tanh", ] [gen_zipped_test_file( diff --git a/tensorflow/lite/testing/generate_examples_lib.py b/tensorflow/lite/testing/generate_examples_lib.py index db34bd62db0..20f66032d35 100644 --- a/tensorflow/lite/testing/generate_examples_lib.py +++ b/tensorflow/lite/testing/generate_examples_lib.py @@ -54,6 +54,7 @@ from tensorflow.lite.testing.op_tests.constant import make_constant_tests from tensorflow.lite.testing.op_tests.control_dep import make_control_dep_tests from tensorflow.lite.testing.op_tests.conv import make_conv_tests from tensorflow.lite.testing.op_tests.conv2d_transpose import make_conv2d_transpose_tests +from tensorflow.lite.testing.op_tests.conv_activation import make_conv_relu_tests, make_conv_relu1_tests, make_conv_relu6_tests # Note: This is a regression test for a bug (b/112303004) that Toco incorrectly # transforms Conv into DepthwiseConv when two Conv ops share the same constant # weight tensor. diff --git a/tensorflow/lite/testing/op_tests/conv_activation.py b/tensorflow/lite/testing/op_tests/conv_activation.py new file mode 100644 index 00000000000..4ee2ae80af7 --- /dev/null +++ b/tensorflow/lite/testing/op_tests/conv_activation.py @@ -0,0 +1,139 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test configs for conv with activations.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.lite.testing.zip_test_utils import create_tensor_data +from tensorflow.lite.testing.zip_test_utils import make_zip_of_tests +from tensorflow.lite.testing.zip_test_utils import register_make_test_function + + +def make_conv_activation_tests(activation_op): + """Make a set of tests to do convolution with activation.""" + + def f(options): + """Actual function that generates examples.""" + test_parameters = [ + { + "input_shape": [[1, 3, 4, 3], [4, 6, 6, 1]], + "filter_shape": [[1, 1], [2, 3], [3, 3]], + "strides": [[1, 1, 1, 1], [1, 2, 3, 1]], + "dilations": [[1, 1, 1, 1], [1, 3, 2, 1], [1, 2, 2, 1]], + "padding": ["SAME", "VALID"], + "data_format": ["NHWC"], # TODO(aselle): NCHW would be good + "constant_filter": [True, False], + "channel_multiplier": [1, 2], + "fully_quantize": [False], + }, + # TODO(b/134702301): The fully_quantize param is just ignored by the + # MLIR testing path now, resulting in duplicate tests. Either ignore + # these tests or handle it properly in the mlir_convert() function. + { + "input_shape": [[1, 3, 4, 3], [4, 6, 6, 1]], + "filter_shape": [[1, 1], [2, 3], [3, 3]], + "strides": [[1, 1, 1, 1], [1, 2, 3, 1]], + "dilations": [[1, 1, 1, 1], [1, 3, 2, 1], [1, 2, 2, 1]], + "padding": ["SAME", "VALID"], + "data_format": ["NHWC"], # TODO(aselle): NCHW would be good + "constant_filter": [True], + "channel_multiplier": [1, 2], + "fully_quantize": [True], + } + ] + + def get_tensor_shapes(parameters): + input_shape = parameters["input_shape"] + filter_size = parameters["filter_shape"] + filter_shape = filter_size + [ + input_shape[3], parameters["channel_multiplier"] + ] + return [input_shape, filter_shape] + + def build_graph(parameters): + """Build a conv graph given `parameters`.""" + input_shape, filter_shape = get_tensor_shapes(parameters) + input_tensor = tf.compat.v1.placeholder( + dtype=tf.float32, name="input", shape=input_shape) + + # Get filter input either as a placeholder or constants. Also get a list + # of the input tensors that are represented as placeholders. + if parameters["constant_filter"]: + filter_input = create_tensor_data( + np.float32, filter_shape, min_value=-10, max_value=10) + input_tensors = [input_tensor] + else: + filter_input = tf.compat.v1.placeholder( + dtype=tf.float32, name="filter", shape=filter_shape) + input_tensors = [input_tensor, filter_input] + + out = tf.nn.conv2d( + input_tensor, + filter_input, + strides=parameters["strides"], + dilations=parameters["dilations"], + padding=parameters["padding"], + data_format=parameters["data_format"]) + out = activation_op(out) + return input_tensors, [out] + + def build_inputs(parameters, sess, inputs, outputs): + """Build inputs for conv with activation.""" + + input_shape, filter_shape = get_tensor_shapes(parameters) + values = [ + create_tensor_data( + np.float32, input_shape, min_value=-1, max_value=1) + ] + if not parameters["constant_filter"]: + values.append(create_tensor_data(np.float32, filter_shape)) + return values, sess.run(outputs, feed_dict=dict(zip(inputs, values))) + + make_zip_of_tests( + options, + test_parameters, + build_graph, + build_inputs, + expected_tf_failures=60) + + return f + + +@register_make_test_function() +def make_conv_relu6_tests(options): + """Make a set of tests to do conv_relu6.""" + return make_conv_activation_tests(tf.nn.relu6)(options) + + +@register_make_test_function() +def make_conv_relu_tests(options): + """Make a set of tests to do conv_relu.""" + return make_conv_activation_tests(tf.nn.relu)(options) + + +def relu1(input_tensor): + # Note that the following is not supported: + # out = tf.maximum(-1.0, tf.minimum(input_tensor, 1.0)) + out = tf.minimum(1.0, tf.maximum(input_tensor, -1.0)) + return out + + +@register_make_test_function() +def make_conv_relu1_tests(options): + """Make a set of tests to do conv_relu1.""" + return make_conv_activation_tests(relu1)(options) diff --git a/tensorflow/lite/testing/op_tests/tanh.py b/tensorflow/lite/testing/op_tests/tanh.py index 1eba70bf7bf..96f306f60a7 100644 --- a/tensorflow/lite/testing/op_tests/tanh.py +++ b/tensorflow/lite/testing/op_tests/tanh.py @@ -32,6 +32,8 @@ def make_tanh_tests(options): test_parameters = [{ "input_shape": [[], [1], [2, 3], [1, 1, 1, 1], [1, 3, 4, 3], [3, 15, 14, 3], [3, 1, 2, 4, 6], [2, 2, 3, 4, 5, 6]], + "fully_quantize": [True, False], + "input_range": [(-4, 10)] }] def build_graph(parameters): @@ -41,8 +43,9 @@ def make_tanh_tests(options): return [input_tensor], [out] def build_inputs(parameters, sess, inputs, outputs): - input_values = create_tensor_data( - np.float32, parameters["input_shape"], min_value=-4, max_value=10) + min_value, max_value = parameters["input_range"] + input_values = create_tensor_data(np.float32, parameters["input_shape"], + min_value, max_value) return [input_values], sess.run( outputs, feed_dict=dict(zip(inputs, [input_values]))) diff --git a/tensorflow/lite/testing/toco_convert.py b/tensorflow/lite/testing/toco_convert.py index e533d7246ce..1411f4df946 100644 --- a/tensorflow/lite/testing/toco_convert.py +++ b/tensorflow/lite/testing/toco_convert.py @@ -104,6 +104,9 @@ def toco_convert(options, graph_def, input_tensors, output_tensors, **kwargs): data_types = [zip_test_utils.TF_TYPE_INFO[x[2]][1] for x in input_tensors] if test_params.get("fully_quantize", False): + # Read the input range for the representative dataset from parameters. + min_value, max_value = test_params.get("input_range", (-1, 1)) + with tempfile.NamedTemporaryFile() as graphdef_file: graphdef_file.write(graph_def_str) graphdef_file.flush() @@ -118,7 +121,8 @@ def toco_convert(options, graph_def, input_tensors, output_tensors, **kwargs): if shape: dims = [dim.value for dim in shape.dims] calibration_inputs.append( - np.random.uniform(-1, 1, tuple(dims)).astype(np.float32)) + np.random.uniform(min_value, max_value, + tuple(dims)).astype(np.float32)) return calibration_inputs def representative_dataset_gen():