diff --git a/tensorflow/lite/testing/generated_examples_zip_test.cc b/tensorflow/lite/testing/generated_examples_zip_test.cc index 16b4675bb0d..d1b3d267eba 100644 --- a/tensorflow/lite/testing/generated_examples_zip_test.cc +++ b/tensorflow/lite/testing/generated_examples_zip_test.cc @@ -156,14 +156,22 @@ const std::map& GetKnownBrokenNnapiTests() { const std::map& GetKnownQuantizeBrokenTests() { static const std::map* const kQuantizeBrokenTests = new std::map({ - {R"(^\/conv.*fully_quantize=True)", "134594898"}, - {R"(^\/depthwiseconv.*fully_quantize=True)", "134594898"}, {R"(^\/sum.*fully_quantize=True)", "134594898"}, {R"(^\/l2norm.*fully_quantize=True)", "134594898"}, }); return *kQuantizeBrokenTests; } +const std::map& GetQuantizeTestsError() { + static const std::map* const kQuantizeBrokenTests = + new std::map({ + {R"(^\/conv_relu1.*fully_quantize=True)", 18}, + {R"(^\/conv_relu6.*fully_quantize=True)", 8}, + {R"(^\/maximum.*fully_quantize=True)", 8}, + }); + return *kQuantizeBrokenTests; +} + // Allows test data to be unarchived into a temporary directory and makes // sure those temporary directories are removed later. class ArchiveEnvironment : public ::testing::Environment { @@ -299,10 +307,16 @@ TEST_P(OpsTest, RunZipTests) { tflite::testing::TfLiteDriver test_driver( FLAGS_use_nnapi ? TfLiteDriver::DelegateType::kNnapi : TfLiteDriver::DelegateType::kNone); + + auto quantized_tests_error = GetQuantizeTestsError(); bool fully_quantize = false; if (test_path.find("fully_quantize=True") != std::string::npos) { - // TODO(b/134594898): Tighten this constraint. - test_driver.SetThreshold(0.2, 0.1); + for (const auto& p : quantized_tests_error) { + if (RE2::PartialMatch(test_name, p.first)) { + test_driver.SetQuantizationErrorMultiplier(p.second); + break; + } + } fully_quantize = true; } @@ -313,7 +327,6 @@ TEST_P(OpsTest, RunZipTests) { auto kBrokenNnapiTests = GetKnownBrokenNnapiTests(); broken_tests.insert(kBrokenNnapiTests.begin(), kBrokenNnapiTests.end()); } - auto quantize_broken_tests = GetKnownQuantizeBrokenTests(); bool result = tflite::testing::ParseAndRunTests(&tflite_stream, &test_driver); string message = test_driver.GetErrorMessage(); @@ -346,7 +359,7 @@ TEST_P(OpsTest, RunZipTests) { if (!result) { string bug_number; // See if the tests are potential quantize failures. - for (const auto& p : quantize_broken_tests) { + for (const auto& p : GetKnownQuantizeBrokenTests()) { if (RE2::PartialMatch(test_name, p.first)) { bug_number = p.second; break; diff --git a/tensorflow/lite/testing/tflite_driver.cc b/tensorflow/lite/testing/tflite_driver.cc index 795fb1fee99..47293016ab6 100644 --- a/tensorflow/lite/testing/tflite_driver.cc +++ b/tensorflow/lite/testing/tflite_driver.cc @@ -18,7 +18,6 @@ limitations under the License. #include #include #include - #include "absl/strings/escaping.h" #include "tensorflow/lite/builtin_op_data.h" #include "tensorflow/lite/delegates/flex/delegate.h" @@ -37,6 +36,22 @@ namespace { const double kRelativeThreshold = 1e-2f; const double kAbsoluteThreshold = 1e-4f; +// For quantized tests, we use a different error measurement from float ones. +// Assumes the baseline is a always a float TF model. +// Error of a quantized model compared to the baseline comes from two sources: +// 1. the math done with quantized inputs, and +// 2. quantization of the output. +// Assumes there is no error introduced by source 1, the theoretical maximum +// error allowed for the output is 0.5 * scale, because scale is equal to the +// size of the quantization bucket. +// +// As a result, we use `scale` as a unit for measuring the quantization error. +// To add the error introduced by source 1 as well, we need to relax the +// multiplier from 0.5 to a larger number, which is model/op dependent. +// The number below is good enough to account for both the two sources of error +// for most quantized op tests to pass. +const int kQuantizationErrorMultiplier = 4; + // Returns the value in the given position in a tensor. template T Value(void* data, int index) { @@ -58,15 +73,31 @@ unique_void_ptr make_type_erased_array(size_t size) { [](void* data) { delete[] static_cast(data); }); } +bool IsQuantized(const TfLiteTensor& tensor) { + if (tensor.type != kTfLiteInt8) return false; + + if (tensor.quantization.params != nullptr) { + auto* quantization = + reinterpret_cast(tensor.quantization.params); + if (quantization->scale != nullptr && quantization->scale->size == 1 && + quantization->zero_point != nullptr && + quantization->zero_point->size == 1) { + return true; + } + } + return false; +} } // namespace class TfLiteDriver::DataExpectation { public: - DataExpectation(double relative_threshold, double absolute_threshold) + DataExpectation(double relative_threshold, double absolute_threshold, + int quantization_error_multiplier) : data_(nullptr, nullptr), num_elements_(0), relative_threshold_(relative_threshold), - absolute_threshold_(absolute_threshold) {} + absolute_threshold_(absolute_threshold), + quantization_error_multiplier_(quantization_error_multiplier) {} template void SetData(const string& csv_values) { @@ -128,11 +159,13 @@ class TfLiteDriver::DataExpectation { } bool TypedCheckString(bool verbose, const TfLiteTensor& tensor); + bool QuantizedCheck(bool verbose, const TfLiteTensor& tensor); unique_void_ptr data_; size_t num_elements_; double relative_threshold_; double absolute_threshold_; + int quantization_error_multiplier_; }; class TfLiteDriver::ShapeExpectation { @@ -218,8 +251,37 @@ bool TfLiteDriver::DataExpectation::TypedCheckString( return true; } +bool TfLiteDriver::DataExpectation::QuantizedCheck(bool verbose, + const TfLiteTensor& tensor) { + auto* quantization = + reinterpret_cast(tensor.quantization.params); + const float scale = quantization->scale->data[0]; + const int32 zero_point = quantization->zero_point->data[0]; + + bool good_result = true; + for (int i = 0; i < tensor.bytes; i++) { + const int32 computed = tensor.data.int8[i]; + const float dequantized = + static_cast(scale * (computed - zero_point)); + const float reference = Value(data_.get(), i); + if (std::abs(dequantized - reference) > + quantization_error_multiplier_ * scale) { + if (verbose) { + std::cerr << " index " << i << ": got " << dequantized + << ", but expected " << reference << std::endl; + } + good_result = false; + } + } + return good_result; +} + bool TfLiteDriver::DataExpectation::Check(bool verbose, const TfLiteTensor& tensor) { + if (IsQuantized(tensor)) { + return QuantizedCheck(verbose, tensor); + } + switch (tensor.type) { case kTfLiteFloat32: return TypedCheck(verbose, tensor); @@ -247,7 +309,8 @@ bool TfLiteDriver::DataExpectation::Check(bool verbose, TfLiteDriver::TfLiteDriver(DelegateType delegate_type, bool reference_kernel) : delegate_(nullptr, nullptr), relative_threshold_(kRelativeThreshold), - absolute_threshold_(kAbsoluteThreshold) { + absolute_threshold_(kAbsoluteThreshold), + quantization_error_multiplier_(kQuantizationErrorMultiplier) { if (reference_kernel) { resolver_.reset(new ops::builtin::BuiltinRefOpResolver); } else { @@ -395,6 +458,11 @@ void TfLiteDriver::SetThreshold(double relative_threshold, absolute_threshold_ = absolute_threshold; } +void TfLiteDriver::SetQuantizationErrorMultiplier( + int quantization_error_multiplier) { + quantization_error_multiplier_ = quantization_error_multiplier; +} + void TfLiteDriver::SetExpectation(int id, const string& csv_values) { if (!IsValid()) return; auto* tensor = interpreter_->tensor(id); @@ -402,7 +470,14 @@ void TfLiteDriver::SetExpectation(int id, const string& csv_values) { Invalidate(absl::StrCat("Overridden expectation for tensor '", id, "'")); } expected_output_[id].reset( - new DataExpectation(relative_threshold_, absolute_threshold_)); + new DataExpectation(relative_threshold_, absolute_threshold_, + quantization_error_multiplier_)); + + if (IsQuantized(*tensor)) { + expected_output_[id]->SetData(csv_values); + return; + } + switch (tensor->type) { case kTfLiteFloat32: expected_output_[id]->SetData(csv_values); diff --git a/tensorflow/lite/testing/tflite_driver.h b/tensorflow/lite/testing/tflite_driver.h index ae843d1cba7..258902606a5 100644 --- a/tensorflow/lite/testing/tflite_driver.h +++ b/tensorflow/lite/testing/tflite_driver.h @@ -64,6 +64,7 @@ class TfLiteDriver : public TestRunner { bool CheckResults() override; string ReadOutput(int id) override; void SetThreshold(double relative_threshold, double absolute_threshold); + void SetQuantizationErrorMultiplier(int quantization_error_multiplier); protected: Interpreter::TfLiteDelegatePtr delegate_; @@ -95,6 +96,7 @@ class TfLiteDriver : public TestRunner { std::map tensors_to_deallocate_; double relative_threshold_; double absolute_threshold_; + int quantization_error_multiplier_; }; } // namespace testing diff --git a/tensorflow/lite/testing/tflite_driver_test.cc b/tensorflow/lite/testing/tflite_driver_test.cc index 99efd2d66d1..6dac9565dde 100644 --- a/tensorflow/lite/testing/tflite_driver_test.cc +++ b/tensorflow/lite/testing/tflite_driver_test.cc @@ -112,7 +112,7 @@ TEST(TfliteDriverTest, AddQuantizedInt8Test) { runner->SetInput(1, "1,1,1,1"); - runner->SetExpectation(2, "3,3,3,3"); + runner->SetExpectation(2, "0.0117,0.0117,0.0117,0.0117"); runner->Invoke(); ASSERT_TRUE(runner->IsValid()); diff --git a/tensorflow/lite/testing/toco_convert.py b/tensorflow/lite/testing/toco_convert.py index 3e8a489c5f8..e8d1e8eec12 100644 --- a/tensorflow/lite/testing/toco_convert.py +++ b/tensorflow/lite/testing/toco_convert.py @@ -146,6 +146,8 @@ def toco_convert(options, graph_def, input_tensors, output_tensors, **kwargs): if extra_toco_options.inference_output_type: converter.inference_output_type = ( extra_toco_options.inference_output_type) + else: + converter.inference_output_type = tf.int8 try: tflite_model = converter.convert()