From 5ef02d3f57c5fc85bbd221972dae0de84f1cc153 Mon Sep 17 00:00:00 2001 From: Taehee Jeong Date: Wed, 16 Dec 2020 17:51:01 -0800 Subject: [PATCH] Fix lite_tests for new quantizer MLIR quantizer test path is added for following tests. * testQuantizeFloat16 * testIntegerQuantizationWithUnsupportedOps PiperOrigin-RevId: 347932208 Change-Id: Iefed1daab7e364621c5f9f6d65a234b0eb85bf44 --- tensorflow/lite/python/lite_test.py | 74 +++++++++++++++++--------- tensorflow/lite/python/lite_v2_test.py | 38 ++++++------- 2 files changed, 68 insertions(+), 44 deletions(-) diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py index 5245248ad9e..ed8059a8b78 100644 --- a/tensorflow/lite/python/lite_test.py +++ b/tensorflow/lite/python/lite_test.py @@ -312,10 +312,14 @@ class FromSessionTest(TestModels, parameterized.TestCase): ('_INT16Quantize_INT16InputOutput', False, True, dtypes.int16), ('_IntOnly_INT8InputOutput', True, False, dtypes.int8), ('_IntOnly_UINT8InputOutput', True, False, dtypes.uint8), - ('_IntOnly_INT16Quantize_INT16InputOutput', True, True, dtypes.int16)) - def testIntegerQuantizationWithUnsupportedOps(self, is_int_only, + ('_IntOnly_INT16Quantize_INT16InputOutput', True, True, dtypes.int16), + ('_IntOnly_INT8InputOutputMlirQuant', True, False, dtypes.int8, True), + ('_IntOnly_UINT8InputOutputMlirQuant', True, False, dtypes.uint8, True)) + def testIntegerQuantizationWithUnsupportedOps(self, + is_int_only, is_int16_quantize, - inference_input_output_type): + inference_input_output_type, + enable_mlir_quantizer=False): with ops.Graph().as_default(): in_tensor_a = array_ops.placeholder(shape=[3], dtype=dtypes.float32) in_tensor_b = array_ops.placeholder(shape=[3], dtype=dtypes.float32) @@ -363,23 +367,25 @@ class FromSessionTest(TestModels, parameterized.TestCase): quantized_converter.inference_input_type = inference_input_output_type quantized_converter.inference_output_type = inference_input_output_type + quantized_converter._experimental_new_quantizer = enable_mlir_quantizer quantized_tflite_model = quantized_converter.convert() self.assertIsNotNone(quantized_tflite_model) + expected_dtype = inference_input_output_type.as_numpy_dtype + # Allow float32 for fallback on non-quantizable op. + expected_ceil_dtype = ( + expected_dtype if enable_mlir_quantizer else dtypes.float32) + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertLen(input_details, 2) - # Allow float32 for fallback. - self.assertEqual(input_details[0]['dtype'], dtypes.float32) - self.assertEqual(input_details[1]['dtype'], - inference_input_output_type.as_numpy_dtype) + self.assertEqual(input_details[0]['dtype'], expected_ceil_dtype) + self.assertEqual(input_details[1]['dtype'], expected_dtype) output_details = interpreter.get_output_details() self.assertLen(output_details, 2) - # Allow float32 for fallback. - self.assertEqual(output_details[0]['dtype'], dtypes.float32) - self.assertEqual(output_details[1]['dtype'], - inference_input_output_type.as_numpy_dtype) + self.assertEqual(output_details[0]['dtype'], expected_ceil_dtype) + self.assertEqual(output_details[1]['dtype'], expected_dtype) @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir @@ -1108,27 +1114,35 @@ class FromSessionTest(TestModels, parameterized.TestCase): @parameterized.named_parameters( # Quantize to Float16 even if rep data provided. - ('UseRepresentativeData', True, False, True, False, False, False), + ('UseRepresentativeData', True, False, True, False, False, False, False), # Quantize to Float16 if no rep data provided. - ('NoRepresentativeData', False, False, True, False, False, False), + ('NoRepresentativeData', False, False, True, False, False, False, False), # Post training quantization if both rep data and int8 included. - ('UseSampleDataIncludeInt8', True, True, False, False, True, False), - + ('UseSampleDataIncludeInt8', True, True, False, False, True, False, False + ), # Quantize to Float16 even if rep data provided with mlir. - ('UseRepresentativeDataMlir', True, False, True, False, False, True), + ('UseRepresentativeDataMlir', True, False, True, False, False, True, False + ), # Quantize to Float16 if no rep data provided with mlir. - ('NoRepresentativeDataMlir', False, False, True, False, False, True), + ('NoRepresentativeDataMlir', False, False, True, False, False, True, False + ), # Post training quantization if both rep data and int8 included with mlir. - ('SampleDataIncludeInt8Mlir', True, True, False, False, True, True)) + ('SampleDataIncludeInt8Mlir', True, True, False, False, True, True, False + ), + # Same as above, but using MLIR quantizer + ('SampleDataIncludeInt8MlirQuant', True, True, False, False, True, True, + True)) def testQuantizeFloat16(self, use_rep_data, include_int8, is_float16_quantized, is_error, - is_post_training_quantized, enable_mlir_converter): + is_post_training_quantized, enable_mlir_converter, + enable_mlir_quantizer): with ops.Graph().as_default(): inp, output, calibration_gen = self._getIntegerQuantizeModel() sess = session.Session() - idx = 1 if enable_mlir_converter else 0 - node_name = 'Conv2D' if enable_mlir_converter else 'Conv2D_bias' + bias_idx = 1 if enable_mlir_converter else 0 + bias_name = 'Conv2D' if enable_mlir_converter else 'Conv2D_bias' + # Convert float model. float_converter = lite.TFLiteConverter.from_session(sess, [inp], [output]) float_converter.experimental_new_converter = enable_mlir_converter @@ -1136,13 +1150,20 @@ class FromSessionTest(TestModels, parameterized.TestCase): self.assertIsNotNone(float_tflite_model) interpreter = Interpreter(model_content=float_tflite_model) interpreter.allocate_tensors() - self.assertEqual(interpreter.get_tensor_details()[idx]['name'], node_name) - self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], + self.assertEqual(interpreter.get_tensor_details()[bias_idx]['name'], + bias_name) + self.assertEqual(interpreter.get_tensor_details()[bias_idx]['dtype'], dtypes.float32) + + # MLIR quantizer has different bias index. + if enable_mlir_quantizer: + bias_idx = 2 + # Convert model to quantized version quantized_converter = lite.TFLiteConverter.from_session( sess, [inp], [output]) quantized_converter.experimental_new_converter = enable_mlir_converter + quantized_converter._experimental_new_quantizer = enable_mlir_quantizer quantized_converter.optimizations = [lite.Optimize.DEFAULT] quantized_converter.target_spec.supported_types = [dtypes.float16] if include_int8: @@ -1162,15 +1183,16 @@ class FromSessionTest(TestModels, parameterized.TestCase): self.assertIsNotNone(quantized_tflite_model) interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() - self.assertEqual(interpreter.get_tensor_details()[idx]['name'], node_name) + self.assertEqual(interpreter.get_tensor_details()[bias_idx]['name'], + bias_name) if is_float16_quantized: # Verify that bias constant is float16 type. - self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], + self.assertEqual(interpreter.get_tensor_details()[bias_idx]['dtype'], dtypes.float16) elif is_post_training_quantized: # Verify that bias constants is int32 type. - self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], + self.assertEqual(interpreter.get_tensor_details()[bias_idx]['dtype'], dtypes.int32) else: raise ValueError('Invalid test options.') diff --git a/tensorflow/lite/python/lite_v2_test.py b/tensorflow/lite/python/lite_v2_test.py index 945e3da8251..b0cea7d1306 100644 --- a/tensorflow/lite/python/lite_v2_test.py +++ b/tensorflow/lite/python/lite_v2_test.py @@ -314,9 +314,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): converter = lite.TFLiteConverterV2.from_concrete_functions([func]) # TODO(b/156309549): We should add INT16 to the builtin types. converter.optimizations = [lite.Optimize.DEFAULT] - converter.target_spec.supported_ops = [ - lite.OpsSet.TFLITE_BUILTINS_INT8 - ] + converter.target_spec.supported_ops = [lite.OpsSet.TFLITE_BUILTINS_INT8] converter.representative_dataset = calibration_gen converter._experimental_calibrate_only = True calibrated_tflite = converter.convert() @@ -610,11 +608,15 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): ('_INT16Quantize_INT16InputOutput', False, True, dtypes.int16), ('_IntOnly_INT8InputOutput', True, False, dtypes.int8), ('_IntOnly_UINT8InputOutput', True, False, dtypes.uint8), - ('_IntOnly_INT16Quantize_INT16InputOutput', True, True, dtypes.int16)) + ('_IntOnly_INT16Quantize_INT16InputOutput', True, True, dtypes.int16), + ('_IntOnly_INT8InputOutputMlirQuant', True, False, dtypes.int8, True), + ('_IntOnly_UINT8InputOutputMlirQuant', True, False, dtypes.uint8, True)) @test_util.run_v2_only - def testIntegerQuantizationWithUnsupportedOps(self, is_int_only, + def testIntegerQuantizationWithUnsupportedOps(self, + is_int_only, is_int16_quantize, - inference_input_output_type): + inference_input_output_type, + enable_mlir_quantizer=False): func, calib_gen = self._getIntegerQuantizationModelWithUnsupportedOps() quantized_converter = tf.lite.TFLiteConverter.from_concrete_functions( @@ -646,23 +648,25 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): quantized_converter.inference_input_type = inference_input_output_type quantized_converter.inference_output_type = inference_input_output_type + quantized_converter._experimental_new_quantizer = enable_mlir_quantizer quantized_tflite_model = quantized_converter.convert() self.assertIsNotNone(quantized_tflite_model) + expected_dtype = inference_input_output_type.as_numpy_dtype + # Allow float32 for fallback on non-quantizable op. + expected_ceil_dtype = ( + expected_dtype if enable_mlir_quantizer else dtypes.float32) + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertLen(input_details, 2) - # Allow float32 for fallback. - self.assertEqual(input_details[0]['dtype'], dtypes.float32) - self.assertEqual(input_details[1]['dtype'], - inference_input_output_type.as_numpy_dtype) + self.assertEqual(input_details[0]['dtype'], expected_ceil_dtype) + self.assertEqual(input_details[1]['dtype'], expected_dtype) output_details = interpreter.get_output_details() self.assertLen(output_details, 2) - # Allow float32 for fallback. - self.assertEqual(output_details[0]['dtype'], dtypes.float32) - self.assertEqual(output_details[1]['dtype'], - inference_input_output_type.as_numpy_dtype) + self.assertEqual(output_details[0]['dtype'], expected_ceil_dtype) + self.assertEqual(output_details[1]['dtype'], expected_dtype) class FromSavedModelTest(lite_v2_test_util.ModelTest): @@ -921,8 +925,7 @@ class FromSavedModelTest(lite_v2_test_util.ModelTest): self.assertEqual(len(signature_defs.values()), 1) self.assertEqual( list(signature_defs['mul_add'].keys()), ['inputs', 'outputs']) - self.assertEqual( - sorted(signature_defs['mul_add']['inputs']), ['x', 'y']) + self.assertCountEqual(signature_defs['mul_add']['inputs'], ['x', 'y']) self.assertEqual(list(signature_defs['mul_add']['outputs']), ['output_0']) @test_util.run_v2_only @@ -962,8 +965,7 @@ class FromSavedModelTest(lite_v2_test_util.ModelTest): self.assertEqual(len(signature_defs.values()), 1) self.assertEqual( list(signature_defs['mul_add'].keys()), ['inputs', 'outputs']) - self.assertEqual( - sorted(signature_defs['mul_add']['inputs']), ['x', 'y']) + self.assertCountEqual(signature_defs['mul_add']['inputs'], ['x', 'y']) self.assertEqual(list(signature_defs['mul_add']['outputs']), ['output_0']) @test_util.run_v2_only