diff --git a/tensorflow/lite/micro/examples/image_recognition_experimental/image_recognition_test.cc b/tensorflow/lite/micro/examples/image_recognition_experimental/image_recognition_test.cc index d4dfee41a8b..fd547b433ef 100644 --- a/tensorflow/lite/micro/examples/image_recognition_experimental/image_recognition_test.cc +++ b/tensorflow/lite/micro/examples/image_recognition_experimental/image_recognition_test.cc @@ -53,7 +53,7 @@ TF_LITE_MICRO_TEST(TestImageRecognitionInvoke) { micro_op_resolver.AddBuiltin(tflite::BuiltinOperator_SOFTMAX, tflite::ops::micro::Register_SOFTMAX()); - const int tensor_arena_size = 45 * 1024; + const int tensor_arena_size = 50 * 1024; uint8_t tensor_arena[tensor_arena_size]; tflite::MicroInterpreter interpreter(model, micro_op_resolver, tensor_arena, diff --git a/tensorflow/lite/micro/examples/image_recognition_experimental/main.cc b/tensorflow/lite/micro/examples/image_recognition_experimental/main.cc index 613c97f15be..09c76df0379 100644 --- a/tensorflow/lite/micro/examples/image_recognition_experimental/main.cc +++ b/tensorflow/lite/micro/examples/image_recognition_experimental/main.cc @@ -67,7 +67,7 @@ int main(int argc, char** argv) { micro_op_resolver.AddBuiltin(tflite::BuiltinOperator_SOFTMAX, tflite::ops::micro::Register_SOFTMAX()); - constexpr int tensor_arena_size = 45 * 1024; + constexpr int tensor_arena_size = 50 * 1024; uint8_t tensor_arena[tensor_arena_size]; tflite::MicroInterpreter interpreter(model, resolver, tensor_arena, tensor_arena_size, error_reporter); diff --git a/tensorflow/lite/micro/examples/person_detection/main_functions.cc b/tensorflow/lite/micro/examples/person_detection/main_functions.cc index 279b09e4c2b..0e5c6394d56 100644 --- a/tensorflow/lite/micro/examples/person_detection/main_functions.cc +++ b/tensorflow/lite/micro/examples/person_detection/main_functions.cc @@ -34,7 +34,7 @@ tflite::MicroInterpreter* interpreter = nullptr; TfLiteTensor* input = nullptr; // An area of memory to use for input, output, and intermediate arrays. -constexpr int kTensorArenaSize = 73 * 1024; +constexpr int kTensorArenaSize = 93 * 1024; static uint8_t tensor_arena[kTensorArenaSize]; } // namespace diff --git a/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc b/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc index 51a61881ead..8acb93ced17 100644 --- a/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc +++ b/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc @@ -27,7 +27,7 @@ limitations under the License. #include "tensorflow/lite/version.h" // Create an area of memory to use for input, output, and intermediate arrays. -constexpr int tensor_arena_size = 73 * 1024; +constexpr int tensor_arena_size = 93 * 1024; uint8_t tensor_arena[tensor_arena_size]; TF_LITE_MICRO_TESTS_BEGIN diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc b/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc index 719f16b2d36..92d2c091f55 100644 --- a/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc +++ b/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc @@ -41,7 +41,7 @@ TfLiteTensor* input = nullptr; // signed value. // An area of memory to use for input, output, and intermediate arrays. -constexpr int kTensorArenaSize = 125 * 1024; +constexpr int kTensorArenaSize = 136 * 1024; static uint8_t tensor_arena[kTensorArenaSize]; } // namespace diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc b/tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc index b0979735d4f..c3719e559ca 100644 --- a/tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc +++ b/tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc @@ -27,7 +27,7 @@ limitations under the License. #include "tensorflow/lite/version.h" // Create an area of memory to use for input, output, and intermediate arrays. -constexpr int tensor_arena_size = 125 * 1024; +constexpr int tensor_arena_size = 136 * 1024; uint8_t tensor_arena[tensor_arena_size]; TF_LITE_MICRO_TESTS_BEGIN diff --git a/tensorflow/lite/micro/kernels/conv.cc b/tensorflow/lite/micro/kernels/conv.cc index 7334cf13e49..ff425e90ee8 100644 --- a/tensorflow/lite/micro/kernels/conv.cc +++ b/tensorflow/lite/micro/kernels/conv.cc @@ -33,7 +33,6 @@ constexpr int kInputTensor = 0; constexpr int kFilterTensor = 1; constexpr int kBiasTensor = 2; constexpr int kOutputTensor = 0; -constexpr int kMaxChannels = 1024; // Conv is quantized along dimension 0: // https://www.tensorflow.org/lite/performance/quantization_spec @@ -49,9 +48,8 @@ struct OpData { int output_shift; // Per channel output multiplier and shift. - // TODO(b/141139247): Allocate these dynamically when possible. - int32_t per_channel_output_multiplier[kMaxChannels]; - int32_t per_channel_output_shift[kMaxChannels]; + int32_t* per_channel_output_multiplier; + int32_t* per_channel_output_shift; // The range of the fused activation layer. For example for kNone and // uint8_t these would be 0 and 255. @@ -72,10 +70,10 @@ inline PaddingType RuntimePaddingType(TfLitePadding padding) { } TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, int width, int height, - int filter_width, int filter_height, int out_width, - int out_height, const TfLiteType data_type, - OpData* data) { + const TfLiteConvParams* params, int width, + int height, int filter_width, int filter_height, + int out_width, int out_height, + const TfLiteType data_type, OpData* data) { bool has_bias = node->inputs->size == 3; // Check number of inputs/outputs TF_LITE_ENSURE(context, has_bias || node->inputs->size == 2); @@ -109,8 +107,69 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, return kTfLiteOk; } +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + void* data = nullptr; + if (context->AllocatePersistentBuffer(context, sizeof(OpData), &data) == + kTfLiteError) { + return nullptr; + } + return data; +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); + + OpData* data = static_cast(node->user_data); + const auto params = static_cast(node->builtin_data); + + TfLiteTensor* output = GetOutput(context, node, kOutputTensor); + const TfLiteTensor* input = GetInput(context, node, kInputTensor); + const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); + + int input_width = input->dims->data[2]; + int input_height = input->dims->data[1]; + int filter_width = filter->dims->data[2]; + int filter_height = filter->dims->data[1]; + int output_width = output->dims->data[2]; + int output_height = output->dims->data[1]; + + // Dynimically allocate per-channel quantization parameters. + const int num_channels = filter->dims->data[kConvQuantizedDimension]; + TF_LITE_ENSURE_STATUS(context->AllocatePersistentBuffer( + context, num_channels * sizeof(int32_t), + reinterpret_cast(&data->per_channel_output_multiplier))); + TF_LITE_ENSURE_STATUS(context->AllocatePersistentBuffer( + context, num_channels * sizeof(int32_t), + reinterpret_cast(&data->per_channel_output_shift))); + + // All per-channel quantized tensors need valid zero point and scale arrays. + if (input->type == kTfLiteInt8) { + TF_LITE_ENSURE_EQ(context, filter->quantization.type, + kTfLiteAffineQuantization); + + const auto* affine_quantization = + static_cast(filter->quantization.params); + TF_LITE_ENSURE(context, affine_quantization); + TF_LITE_ENSURE(context, affine_quantization->scale); + TF_LITE_ENSURE(context, affine_quantization->zero_point); + + TF_LITE_ENSURE(context, + affine_quantization->scale->size == 1 || + affine_quantization->scale->size == + filter->dims->data[kConvQuantizedDimension]); + TF_LITE_ENSURE_EQ(context, affine_quantization->scale->size, + affine_quantization->zero_point->size); + } + + return CalculateOpData(context, node, params, input_width, input_height, + filter_width, filter_height, output_width, + output_height, input->type, data); +} // namespace conv + void EvalQuantized(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, + TfLiteConvParams* params, const OpData& data, const TfLiteTensor* input, const TfLiteTensor* filter, const TfLiteTensor* bias, TfLiteTensor* im2col, TfLiteTensor* hwcn_weights, TfLiteTensor* output) { @@ -118,10 +177,11 @@ void EvalQuantized(TfLiteContext* context, TfLiteNode* node, const int32_t filter_offset = -filter->params.zero_point; const int32_t output_offset = output->params.zero_point; + // TODO(b/154032858): Investigate removing extra copies. ConvParams op_params; op_params.padding_type = RuntimePaddingType(params->padding); - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; + op_params.padding_values.width = data.padding.width; + op_params.padding_values.height = data.padding.height; op_params.stride_width = params->stride_width; op_params.stride_height = params->stride_height; op_params.dilation_width_factor = params->dilation_width_factor; @@ -129,10 +189,10 @@ void EvalQuantized(TfLiteContext* context, TfLiteNode* node, op_params.input_offset = input_offset; op_params.weights_offset = filter_offset; op_params.output_offset = output_offset; - op_params.output_multiplier = data->output_multiplier; - op_params.output_shift = -data->output_shift; - op_params.quantized_activation_min = data->output_activation_min; - op_params.quantized_activation_max = data->output_activation_max; + op_params.output_multiplier = data.output_multiplier; + op_params.output_shift = -data.output_shift; + op_params.quantized_activation_min = data.output_activation_min; + op_params.quantized_activation_max = data.output_activation_max; reference_ops::Conv(op_params, GetTensorShape(input), GetTensorData(input), GetTensorShape(filter), GetTensorData(filter), GetTensorShape(bias), @@ -142,11 +202,12 @@ void EvalQuantized(TfLiteContext* context, TfLiteNode* node, } void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, + TfLiteConvParams* params, const OpData& data, const TfLiteTensor* input, const TfLiteTensor* filter, const TfLiteTensor* bias, TfLiteTensor* output, TfLiteTensor* im2col) { + // TODO(b/154032858): Investigate removing extra copies. ConvParams op_params; op_params.input_offset = -input->params.zero_point; op_params.output_offset = output->params.zero_point; @@ -154,14 +215,14 @@ void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, op_params.stride_width = params->stride_width; op_params.dilation_height_factor = params->dilation_height_factor; op_params.dilation_width_factor = params->dilation_width_factor; - op_params.padding_values.height = data->padding.height; - op_params.padding_values.width = data->padding.width; - op_params.quantized_activation_min = data->output_activation_min; - op_params.quantized_activation_max = data->output_activation_max; + op_params.padding_values.height = data.padding.height; + op_params.padding_values.width = data.padding.width; + op_params.quantized_activation_min = data.output_activation_min; + op_params.quantized_activation_max = data.output_activation_max; reference_integer_ops::ConvPerChannel( - op_params, data->per_channel_output_multiplier, - data->per_channel_output_shift, GetTensorShape(input), + op_params, data.per_channel_output_multiplier, + data.per_channel_output_shift, GetTensorShape(input), GetTensorData(input), GetTensorShape(filter), GetTensorData(filter), GetTensorShape(bias), GetTensorData(bias), GetTensorShape(output), @@ -169,18 +230,18 @@ void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, } void EvalFloat(TfLiteContext* context, TfLiteNode* node, - TfLiteConvParams* params, OpData* data, + TfLiteConvParams* params, const OpData& data, const TfLiteTensor* input, const TfLiteTensor* filter, const TfLiteTensor* bias, TfLiteTensor* im2col, TfLiteTensor* hwcn_weights, TfLiteTensor* output) { float output_activation_min, output_activation_max; CalculateActivationRange(params->activation, &output_activation_min, &output_activation_max); - + // TODO(b/154032858): Investigate removing extra copies. ConvParams op_params; op_params.padding_type = RuntimePaddingType(params->padding); - op_params.padding_values.width = data->padding.width; - op_params.padding_values.height = data->padding.height; + op_params.padding_values.width = data.padding.width; + op_params.padding_values.height = data.padding.height; op_params.stride_width = params->stride_width; op_params.stride_height = params->stride_height; op_params.dilation_width_factor = params->dilation_width_factor; @@ -204,50 +265,20 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); const TfLiteTensor* bias = GetOptionalInputTensor(context, node, kBiasTensor); - int input_width = input->dims->data[2]; - int input_height = input->dims->data[1]; - int filter_width = filter->dims->data[2]; - int filter_height = filter->dims->data[1]; - int output_width = output->dims->data[2]; - int output_height = output->dims->data[1]; - - OpData data; - - // All per-channel quantized tensors need valid zero point and scale arrays. - if (input->type == kTfLiteInt8) { - TF_LITE_ENSURE_EQ(context, filter->quantization.type, - kTfLiteAffineQuantization); - - const auto* affine_quantization = - reinterpret_cast( - filter->quantization.params); - TF_LITE_ENSURE(context, affine_quantization); - TF_LITE_ENSURE(context, affine_quantization->scale); - TF_LITE_ENSURE(context, affine_quantization->zero_point); - - TF_LITE_ENSURE(context, - affine_quantization->scale->size == 1 || - affine_quantization->scale->size == - filter->dims->data[kConvQuantizedDimension]); - TF_LITE_ENSURE_EQ(context, affine_quantization->scale->size, - affine_quantization->zero_point->size); - } - - TF_LITE_ENSURE_STATUS(CalculateOpData( - context, node, params, input_width, input_height, filter_width, - filter_height, output_width, output_height, input->type, &data)); + TFLITE_DCHECK(node->user_data != nullptr); + const OpData& data = *(static_cast(node->user_data)); switch (input->type) { // Already know in/out types are same. case kTfLiteFloat32: - EvalFloat(context, node, params, &data, input, filter, bias, nullptr, + EvalFloat(context, node, params, data, input, filter, bias, nullptr, nullptr, output); break; case kTfLiteInt8: - EvalQuantizedPerChannel(context, node, params, &data, input, filter, bias, + EvalQuantizedPerChannel(context, node, params, data, input, filter, bias, output, nullptr); break; case kTfLiteUInt8: - EvalQuantized(context, node, params, &data, input, filter, bias, nullptr, + EvalQuantized(context, node, params, data, input, filter, bias, nullptr, nullptr, output); break; default: @@ -261,9 +292,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace conv TfLiteRegistration* Register_CONV_2D() { - static TfLiteRegistration r = {/*init=*/nullptr, + static TfLiteRegistration r = {/*init=*/conv::Init, /*free=*/nullptr, - /*prepare=*/nullptr, + /*prepare=*/conv::Prepare, /*invoke=*/conv::Eval, /*profiling_string=*/nullptr, /*builtin_code=*/0, diff --git a/tensorflow/lite/micro/kernels/xtensa_hifimini/conv.cc b/tensorflow/lite/micro/kernels/xtensa_hifimini/conv.cc index 03eba5082af..8895ccf52d7 100644 --- a/tensorflow/lite/micro/kernels/xtensa_hifimini/conv.cc +++ b/tensorflow/lite/micro/kernels/xtensa_hifimini/conv.cc @@ -185,9 +185,6 @@ inline void Conv1x32Input32x32Filter( ae_q56s acc_56 = AE_ZEROQ56(); const int8_t* input_vals_ptr = input_data - 2; for (int i = 0; i < kFilterDepth; i += 2) { - // Find current input index, minus 2 for Xtensa load - // alignments: - // Load signed 2x 8bit values and right shift into 24bit // alignment: ae_p24x2s input_vals_24x2; @@ -244,7 +241,6 @@ constexpr int kInputTensor = 0; constexpr int kFilterTensor = 1; constexpr int kBiasTensor = 2; constexpr int kOutputTensor = 0; -constexpr int kMaxChannels = 32; // Conv is quantized along dimension 0: // https://www.tensorflow.org/lite/performance/quantization_spec @@ -258,9 +254,8 @@ struct OpData { int output_shift; // Per channel output multiplier and shift. - // TODO(b/141139247): Allocate these dynamically when possible. - int32_t per_channel_output_multiplier[kMaxChannels]; - int32_t per_channel_output_shift[kMaxChannels]; + int32_t* per_channel_output_multiplier; + int32_t* per_channel_output_shift; // The range of the fused activation layer. For example for kNone and // uint8_t these would be 0 and 255. @@ -268,12 +263,6 @@ struct OpData { int32_t output_activation_max; }; -// These constants represent constants specific to the music detect model. -// They exist until (b/132070898) is fixed. -static const int kMaxOpDataSize = 6; -static int op_data_counter = 0; -static OpData kStaticOpData[kMaxOpDataSize]; - TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, TfLiteConvParams* params, int width, int height, int filter_width, int filter_height, int out_width, @@ -301,30 +290,37 @@ TfLiteStatus CalculateOpData(TfLiteContext* context, TfLiteNode* node, TfLiteTensor* output = GetOutput(context, node, kOutputTensor); int output_channels = filter->dims->data[kConvQuantizedDimension]; - TF_LITE_ENSURE_STATUS(tflite::PopulateConvolutionQuantizationParams( + return tflite::PopulateConvolutionQuantizationParams( context, input, filter, bias, output, params->activation, &data->output_multiplier, &data->output_shift, &data->output_activation_min, &data->output_activation_max, data->per_channel_output_multiplier, reinterpret_cast(data->per_channel_output_shift), - output_channels)); + output_channels); } return kTfLiteOk; } -void Free(TfLiteContext* context, void* buffer) { op_data_counter = 0; } +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + void* data = nullptr; + if (context->AllocatePersistentBuffer(context, sizeof(OpData), &data) == + kTfLiteError) { + return nullptr; + } + return data; +} TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); auto* params = reinterpret_cast(node->builtin_data); TfLiteTensor* output = GetOutput(context, node, kOutputTensor); const TfLiteTensor* input = GetInput(context, node, kInputTensor); const TfLiteTensor* filter = GetInput(context, node, kFilterTensor); - // TODO(b/132070898): Use statically slotted OpData structures until a - // scratch memory API is ready. - OpData* op_data = &kStaticOpData[op_data_counter++]; - node->user_data = op_data; + auto* op_data = reinterpret_cast(node->user_data); int input_width = input->dims->data[2]; int input_height = input->dims->data[1]; @@ -333,6 +329,17 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { int output_width = output->dims->data[2]; int output_height = output->dims->data[1]; + // Per channel quantization is only needed for int8 inference. For other + // quantized types, only a single scale and zero point is needed. + const int num_channels = filter->dims->data[kConvQuantizedDimension]; + // Dynimically allocate per-channel quantization parameters. + TF_LITE_ENSURE_STATUS(context->AllocatePersistentBuffer( + context, num_channels * sizeof(int32_t), + reinterpret_cast(&op_data->per_channel_output_multiplier))); + TF_LITE_ENSURE_STATUS(context->AllocatePersistentBuffer( + context, num_channels * sizeof(int32_t), + reinterpret_cast(&op_data->per_channel_output_shift))); + // All per-channel quantized tensors need valid zero point and scale arrays. if (input->type == kTfLiteInt8) { TF_LITE_ENSURE_EQ(context, filter->quantization.type, @@ -353,11 +360,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { affine_quantization->zero_point->size); } - TF_LITE_ENSURE_STATUS(CalculateOpData( - context, node, params, input_width, input_height, filter_width, - filter_height, output_width, output_height, input->type, op_data)); - - return kTfLiteOk; + return CalculateOpData(context, node, params, input_width, input_height, + filter_width, filter_height, output_width, + output_height, input->type, op_data); } void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, @@ -366,6 +371,7 @@ void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, const TfLiteTensor* filter, const TfLiteTensor* bias, TfLiteTensor* output, TfLiteTensor* im2col) { + // TODO(b/154032858): Investigate removing extra copies. ConvParams op_params; op_params.input_offset = -input->params.zero_point; op_params.output_offset = output->params.zero_point; @@ -388,6 +394,8 @@ void EvalQuantizedPerChannel(TfLiteContext* context, TfLiteNode* node, } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + TFLITE_DCHECK(node->user_data != nullptr); + TFLITE_DCHECK(node->builtin_data != nullptr); auto* params = reinterpret_cast(node->builtin_data); auto* op_data = reinterpret_cast(node->user_data); @@ -429,8 +437,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { } // namespace conv TfLiteRegistration* Register_CONV_2D() { - static TfLiteRegistration r = {/*init=*/nullptr, - /*free=*/conv::Free, + static TfLiteRegistration r = {/*init=*/conv::Init, + /*free=*/nullptr, /*prepare=*/conv::Prepare, /*invoke=*/conv::Eval, /*profiling_string=*/nullptr,