From 2658a4a20cdac85442460861689f3b7a3b009926 Mon Sep 17 00:00:00 2001 From: Sachin Joglekar Date: Mon, 27 Apr 2020 16:00:25 -0700 Subject: [PATCH] Redo:Adds support for half_pixel_centers & align_corners in TFLite's resizeNN kernel. (Fix MacOS error) PiperOrigin-RevId: 308716383 Change-Id: I577a87b9527a006650f3173dbfd0f0499bc72fb5 --- .../internal/optimized/optimized_ops.h | 13 +- .../reference/resize_nearest_neighbor.h | 36 ++- .../internal/resize_nearest_neighbor_test.cc | 221 +++++++++++++++++- tensorflow/lite/kernels/internal/types.h | 1 + .../lite/kernels/resize_nearest_neighbor.cc | 1 + .../micro/kernels/resize_nearest_neighbor.cc | 1 + 6 files changed, 257 insertions(+), 16 deletions(-) diff --git a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h index 38bd7bd4057..6e1f805f7f4 100644 --- a/tensorflow/lite/kernels/internal/optimized/optimized_ops.h +++ b/tensorflow/lite/kernels/internal/optimized/optimized_ops.h @@ -30,6 +30,7 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/common.h" #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/internal/reference/add.h" +#include "tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h" #if defined(TF_LITE_USE_CBLAS) && defined(__APPLE__) #include @@ -5915,13 +5916,21 @@ inline void TransposeConvV2( // Integer-only version of ResizeNearestNeighbor. Since scales are represented // in fixed-point and thus approximated, |in_x| or |in_y| may differ from the // reference version. Debug checks are in place to test if this occurs. +// NOTE: If align_corners or half_pixel_centers is true, we use the reference +// version. inline void ResizeNearestNeighbor( const tflite::ResizeNearestNeighborParams& op_params, const RuntimeShape& unextended_input_shape, const uint8* input_data, const RuntimeShape& output_size_shape, const int32* output_size_data, const RuntimeShape& unextended_output_shape, uint8* output_data) { - // Align corners = true is not supported. - TFLITE_DCHECK(!op_params.align_corners); + if (op_params.align_corners || op_params.half_pixel_centers) { + // TODO(b/149823713): Add support for align_corners & half_pixel_centers in + // this kernel. + reference_ops::ResizeNearestNeighbor( + op_params, unextended_input_shape, input_data, output_size_shape, + output_size_data, unextended_output_shape, output_data); + return; + } TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4); TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4); diff --git a/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h b/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h index 25623ca9cff..ed87863a7e5 100644 --- a/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h +++ b/tensorflow/lite/kernels/internal/reference/resize_nearest_neighbor.h @@ -23,14 +23,32 @@ namespace tflite { namespace reference_ops { +inline int32 GetNearestNeighbor(const int input_value, const int32 input_size, + const int32 output_size, + const bool align_corners, + const bool half_pixel_centers) { + const float scale = + (align_corners && output_size > 1) + ? (input_size - 1) / static_cast(output_size - 1) + : input_size / static_cast(output_size); + const float offset = half_pixel_centers ? 0.5f : 0.0f; + int32 output_value = std::min( + align_corners + ? static_cast(std::round((input_value + offset) * scale)) + : static_cast(std::floor((input_value + offset) * scale)), + input_size - 1); + if (half_pixel_centers) { + output_value = std::max(static_cast(0), output_value); + } + return output_value; +} + template inline void ResizeNearestNeighbor( const tflite::ResizeNearestNeighborParams& op_params, const RuntimeShape& unextended_input_shape, const T* input_data, const RuntimeShape& output_size_shape, const int32* output_size_data, const RuntimeShape& unextended_output_shape, T* output_data) { - // Align corners = true is not supported. - TFLITE_DCHECK(!op_params.align_corners); TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4); TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4); @@ -50,10 +68,6 @@ inline void ResizeNearestNeighbor( int32 output_height = output_size_data[0]; int32 output_width = output_size_data[1]; - // We use float to ensure agreement with the Tensorflow implementation. - const float height_scale = static_cast(input_height) / output_height; - const float width_scale = static_cast(input_width) / output_width; - const int col_offset = input_shape.Dims(3); const int row_offset = input_shape.Dims(2) * col_offset; const int batch_offset = input_shape.Dims(1) * row_offset; @@ -62,12 +76,14 @@ inline void ResizeNearestNeighbor( T* output_ptr = output_data; for (int b = 0; b < batches; ++b) { for (int y = 0; y < output_height; ++y) { - int32 in_y = std::min(static_cast(std::floor(y * height_scale)), - input_height - 1); + int32 in_y = GetNearestNeighbor(y, input_height, output_height, + op_params.align_corners, + op_params.half_pixel_centers); const T* y_input_ptr = input_ptr + in_y * row_offset; for (int x = 0; x < output_width; ++x) { - int32 in_x = std::min(static_cast(std::floor(x * width_scale)), - input_width - 1); + int32 in_x = GetNearestNeighbor(x, input_width, output_width, + op_params.align_corners, + op_params.half_pixel_centers); const T* x_input_ptr = y_input_ptr + in_x * col_offset; memcpy(output_ptr, x_input_ptr, depth * sizeof(T)); output_ptr += depth; diff --git a/tensorflow/lite/kernels/internal/resize_nearest_neighbor_test.cc b/tensorflow/lite/kernels/internal/resize_nearest_neighbor_test.cc index 102ee04e6a8..4659d3a80e4 100644 --- a/tensorflow/lite/kernels/internal/resize_nearest_neighbor_test.cc +++ b/tensorflow/lite/kernels/internal/resize_nearest_neighbor_test.cc @@ -30,8 +30,9 @@ void TestReferenceResizeNearestNeighbor( const RuntimeShape& input_shape, const std::vector& input_data, const std::vector& output_size_data, const RuntimeShape& output_shape, - const std::vector& expected_output_data) { - ResizeNearestNeighborParams op_params{/*align_corners=*/false}; + const std::vector& expected_output_data, bool align_corners = false, + bool half_pixel_centers = false) { + ResizeNearestNeighborParams op_params{align_corners, half_pixel_centers}; RuntimeShape output_size_shape({1, 1, 1, 2}); std::vector output_data(expected_output_data.size()); @@ -55,6 +56,30 @@ TEST(ResizeNearestNeighborReference, Test2x2To1x1) { output_shape, output_data); } +TEST(ResizeNearestNeighborReference, Test2x2To1x1_AlignCorners) { + RuntimeShape input_shape = {1, 2, 2, 1}; + std::vector input_data = {1, 2, 3, 4}; + std::vector output_size_data = {1, 1}; + RuntimeShape output_shape = {1, 1, 1, 1}; + std::vector output_data = {1}; + + TestReferenceResizeNearestNeighbor(input_shape, input_data, output_size_data, + output_shape, output_data, + /*align_corners=*/true); +} + +TEST(ResizeNearestNeighborReference, Test2x2To1x1_HalfPixelCenters) { + RuntimeShape input_shape = {1, 2, 2, 1}; + std::vector input_data = {1, 2, 3, 4}; + std::vector output_size_data = {1, 1}; + RuntimeShape output_shape = {1, 1, 1, 1}; + std::vector output_data = {4}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/false, /*half_pixel_centers=*/true); +} + TEST(ResizeNearestNeighborReference, Test2x2To3x3) { RuntimeShape input_shape = {1, 2, 2, 1}; std::vector input_data = {1, 2, 3, 4}; @@ -66,6 +91,30 @@ TEST(ResizeNearestNeighborReference, Test2x2To3x3) { output_shape, output_data); } +TEST(ResizeNearestNeighborReference, Test2x2To3x3_AlignCorners) { + RuntimeShape input_shape = {1, 2, 2, 1}; + std::vector input_data = {1, 2, 3, 4}; + std::vector output_size_data = {3, 3}; + RuntimeShape output_shape = {1, 3, 3, 1}; + std::vector output_data = {1, 2, 2, 3, 4, 4, 3, 4, 4}; + + TestReferenceResizeNearestNeighbor(input_shape, input_data, output_size_data, + output_shape, output_data, + /*align_corners=*/true); +} + +TEST(ResizeNearestNeighborReference, Test2x2To3x3_HalfPixelCenters) { + RuntimeShape input_shape = {1, 2, 2, 1}; + std::vector input_data = {1, 2, 3, 4}; + std::vector output_size_data = {3, 3}; + RuntimeShape output_shape = {1, 3, 3, 1}; + std::vector output_data = {1, 2, 2, 3, 4, 4, 3, 4, 4}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/false, /*half_pixel_centers=*/true); +} + TEST(ResizeNearestNeighborReference, Test3x3To2x2) { RuntimeShape input_shape = {1, 3, 3, 1}; std::vector input_data = {1, 2, 3, 4, 5, 6, 7, 8, 9}; @@ -77,6 +126,30 @@ TEST(ResizeNearestNeighborReference, Test3x3To2x2) { output_shape, output_data); } +TEST(ResizeNearestNeighborReference, Test3x3To2x2_AlignCorners) { + RuntimeShape input_shape = {1, 3, 3, 1}; + std::vector input_data = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + std::vector output_size_data = {2, 2}; + RuntimeShape output_shape = {1, 2, 2, 1}; + std::vector output_data = {1, 3, 7, 9}; + + TestReferenceResizeNearestNeighbor(input_shape, input_data, output_size_data, + output_shape, output_data, + /*align_corners=*/true); +} + +TEST(ResizeNearestNeighborReference, Test3x3To2x2_HalfPixelCenters) { + RuntimeShape input_shape = {1, 3, 3, 1}; + std::vector input_data = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + std::vector output_size_data = {2, 2}; + RuntimeShape output_shape = {1, 2, 2, 1}; + std::vector output_data = {1, 3, 7, 9}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/false, /*half_pixel_centers=*/true); +} + TEST(ResizeNearestNeighborReference, Test2x2To2x5) { RuntimeShape input_shape = {1, 2, 2, 1}; std::vector input_data = {1, 2, 3, 4}; @@ -88,6 +161,18 @@ TEST(ResizeNearestNeighborReference, Test2x2To2x5) { output_shape, output_data); } +TEST(ResizeNearestNeighborReference, Test2x2To2x5_HalfPixelCenters) { + RuntimeShape input_shape = {1, 2, 2, 1}; + std::vector input_data = {1, 2, 3, 4}; + std::vector output_size_data = {2, 5}; + RuntimeShape output_shape = {1, 2, 5, 1}; + std::vector output_data = {1, 1, 2, 2, 2, 3, 3, 4, 4, 4}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/false, /*half_pixel_centers=*/true); +} + TEST(ResizeNearestNeighborReference, Test4x4To3x3) { RuntimeShape input_shape = {1, 4, 4, 1}; std::vector input_data = {1, 2, 3, 4, 5, 6, 7, 8, @@ -100,6 +185,32 @@ TEST(ResizeNearestNeighborReference, Test4x4To3x3) { output_shape, output_data); } +TEST(ResizeNearestNeighborReference, Test4x4To3x3_AlignCorners) { + RuntimeShape input_shape = {1, 4, 4, 1}; + std::vector input_data = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16}; + std::vector output_size_data = {3, 3}; + RuntimeShape output_shape = {1, 3, 3, 1}; + std::vector output_data = {1, 3, 4, 9, 11, 12, 13, 15, 16}; + + TestReferenceResizeNearestNeighbor(input_shape, input_data, output_size_data, + output_shape, output_data, + /*align_corners=*/true); +} + +TEST(ResizeNearestNeighborReference, Test4x4To3x3_HalfPixelCenters) { + RuntimeShape input_shape = {1, 4, 4, 1}; + std::vector input_data = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16}; + std::vector output_size_data = {3, 3}; + RuntimeShape output_shape = {1, 3, 3, 1}; + std::vector output_data = {1, 3, 4, 9, 11, 12, 13, 15, 16}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/false, /*half_pixel_centers=*/true); +} + TEST(ResizeNearestNeighborReference, Test2x2To5x2) { RuntimeShape input_shape = {1, 2, 2, 1}; std::vector input_data = {1, 2, 3, 4}; @@ -111,6 +222,31 @@ TEST(ResizeNearestNeighborReference, Test2x2To5x2) { output_shape, output_data); } +TEST(ResizeNearestNeighborReference, Test2x2To5x2_HalfPixelCenters) { + RuntimeShape input_shape = {1, 2, 2, 1}; + std::vector input_data = {1, 2, 3, 4}; + std::vector output_size_data = {5, 2}; + RuntimeShape output_shape = {1, 5, 2, 1}; + std::vector output_data = {1, 2, 1, 2, 3, 4, 3, 4, 3, 4}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/false, /*half_pixel_centers=*/true); +} + +TEST(ResizeNearestNeighborReference, + Test2x2To5x2_HalfPixelCenters_AlignCorners) { + RuntimeShape input_shape = {1, 2, 2, 1}; + std::vector input_data = {1, 2, 3, 4}; + std::vector output_size_data = {5, 2}; + RuntimeShape output_shape = {1, 5, 2, 1}; + std::vector output_data = {2, 2, 2, 2, 4, 4, 4, 4, 4, 4}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/true, /*half_pixel_centers=*/true); +} + TEST(ResizeNearestNeighborReference, Test2x2To4x4) { RuntimeShape input_shape = {1, 2, 2, 1}; std::vector input_data = {1, 2, 3, 4}; @@ -149,10 +285,56 @@ TEST(ResizeNearestNeighborReference, Test2x2x2x2To2x3x3x2) { output_shape, output_data); } +TEST(ResizeNearestNeighborReference, Test2x2x2x2To2x3x3x2_AlignCorners) { + RuntimeShape input_shape = {2, 2, 2, 2}; + std::vector input_data = {1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8}; + std::vector output_size_data = {3, 3}; + RuntimeShape output_shape = {2, 3, 3, 2}; + std::vector output_data = { + 1, 2, 3, 4, 3, 4, 5, 6, 7, 8, 7, 8, 5, 6, 7, 8, 7, 8, + 1, 2, 3, 4, 3, 4, 5, 6, 7, 8, 7, 8, 5, 6, 7, 8, 7, 8, + }; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/true, /*half_pixel_centers=*/false); +} + +TEST(ResizeNearestNeighborReference, Test2x2x2x2To2x3x3x2_HalfPixelCenters) { + RuntimeShape input_shape = {2, 2, 2, 2}; + std::vector input_data = {1, 1, 2, 2, 3, 3, 4, 4, + 5, 5, 6, 6, 7, 7, 8, 8}; + std::vector output_size_data = {3, 3}; + RuntimeShape output_shape = {2, 3, 3, 2}; + std::vector output_data = {1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 5, 5, 6, 6, 6, 6, + 7, 7, 8, 8, 8, 8, 7, 7, 8, 8, 8, 8}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/false, /*half_pixel_centers=*/true); +} + +TEST(ResizeNearestNeighborReference, + Test2x2x2x2To2x3x3x2_HalfPixelCenters_AlignCorners) { + RuntimeShape input_shape = {2, 2, 2, 2}; + std::vector input_data = {1, 2, 3, 4, 5, 6, 7, 8, + 1, 2, 3, 4, 5, 6, 7, 8}; + std::vector output_size_data = {3, 3}; + RuntimeShape output_shape = {2, 3, 3, 2}; + std::vector output_data = {1, 2, 3, 4, 3, 4, 5, 6, 7, 8, 7, 8, + 5, 6, 7, 8, 7, 8, 1, 2, 3, 4, 3, 4, + 5, 6, 7, 8, 7, 8, 5, 6, 7, 8, 7, 8}; + + TestReferenceResizeNearestNeighbor( + input_shape, input_data, output_size_data, output_shape, output_data, + /*align_corners=*/true, /*half_pixel_centers=*/true); +} + void TestOptimizedResizeNearestNeighbor(int batch, int depth, int input_width, int input_height, int output_width, int output_height) { - ResizeNearestNeighborParams op_params{/*align_corners=*/false}; RuntimeShape output_size_shape({1, 1, 1, 2}); RuntimeShape input_shape({batch, input_height, input_width, depth}); @@ -167,6 +349,9 @@ void TestOptimizedResizeNearestNeighbor(int batch, int depth, int input_width, std::vector output_data(output_shape.FlatSize(), 3); std::vector output_size_data = {output_height, output_width}; + ResizeNearestNeighborParams op_params{/*align_corners=*/false, + /*half_pixel_centers=*/false}; + // Test the optimized version against the reference version. reference_ops::ResizeNearestNeighbor( op_params, input_shape, input_data.data(), output_size_shape, @@ -174,7 +359,35 @@ void TestOptimizedResizeNearestNeighbor(int batch, int depth, int input_width, optimized_ops::ResizeNearestNeighbor( op_params, input_shape, input_data.data(), output_size_shape, output_size_data.data(), output_shape, output_data.data()); + ASSERT_EQ(reference_output_data, output_data); + op_params.align_corners = true; + reference_ops::ResizeNearestNeighbor( + op_params, input_shape, input_data.data(), output_size_shape, + output_size_data.data(), output_shape, reference_output_data.data()); + optimized_ops::ResizeNearestNeighbor( + op_params, input_shape, input_data.data(), output_size_shape, + output_size_data.data(), output_shape, output_data.data()); + ASSERT_EQ(reference_output_data, output_data); + + op_params.align_corners = false; + op_params.half_pixel_centers = true; + reference_ops::ResizeNearestNeighbor( + op_params, input_shape, input_data.data(), output_size_shape, + output_size_data.data(), output_shape, reference_output_data.data()); + optimized_ops::ResizeNearestNeighbor( + op_params, input_shape, input_data.data(), output_size_shape, + output_size_data.data(), output_shape, output_data.data()); + ASSERT_EQ(reference_output_data, output_data); + + op_params.align_corners = true; + op_params.half_pixel_centers = true; + reference_ops::ResizeNearestNeighbor( + op_params, input_shape, input_data.data(), output_size_shape, + output_size_data.data(), output_shape, reference_output_data.data()); + optimized_ops::ResizeNearestNeighbor( + op_params, input_shape, input_data.data(), output_size_shape, + output_size_data.data(), output_shape, output_data.data()); ASSERT_EQ(reference_output_data, output_data); } @@ -214,7 +427,7 @@ bool is_valid_scale(int input_width, int input_height, int output_width, TEST(ResizeNearestNeighborOptimized, TestReferenceParity) { int invalid_count = 0; - const int kTestsToRun = 100 * 1000; + const int kTestsToRun = 10000; for (int i = 0; i < kTestsToRun; i++) { const int batch = ExponentialRandomPositiveInt(0.9f, 3, 20); const int depth = ExponentialRandomPositiveInt(0.9f, 6, 50); diff --git a/tensorflow/lite/kernels/internal/types.h b/tensorflow/lite/kernels/internal/types.h index 0752f5cfa97..cbdedd88901 100644 --- a/tensorflow/lite/kernels/internal/types.h +++ b/tensorflow/lite/kernels/internal/types.h @@ -1007,6 +1007,7 @@ struct ResizeBilinearParams { struct ResizeNearestNeighborParams { bool align_corners; + bool half_pixel_centers; }; struct SliceParams { diff --git a/tensorflow/lite/kernels/resize_nearest_neighbor.cc b/tensorflow/lite/kernels/resize_nearest_neighbor.cc index b783a0e0a67..122f2448d39 100644 --- a/tensorflow/lite/kernels/resize_nearest_neighbor.cc +++ b/tensorflow/lite/kernels/resize_nearest_neighbor.cc @@ -89,6 +89,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::ResizeNearestNeighborParams op_params; op_params.align_corners = params->align_corners; + op_params.half_pixel_centers = false; if (output->type == kTfLiteFloat32) { reference_ops::ResizeNearestNeighbor( diff --git a/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc b/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc index 2bce21e0489..9487e33c45f 100644 --- a/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc +++ b/tensorflow/lite/micro/kernels/resize_nearest_neighbor.cc @@ -67,6 +67,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::ResizeNearestNeighborParams op_params; op_params.align_corners = params->align_corners; + op_params.half_pixel_centers = false; if (output->type == kTfLiteFloat32) { reference_ops::ResizeNearestNeighbor(