experiments/STT-tensorflow
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Create a benchmark for the person detection model.

Browse Source 
PiperOrigin-RevId: 307660913
Change-Id: Ifc63a5c34165efc15d6ac0ec16cc9233bb839bdd
This commit is contained in:
Nat Jeffries 2020-04-21 12:42:38 -07:00 committed by TensorFlower Gardener
parent 86a855ebd4
commit e225b8c8ca
6 changed files with 209 additions and 3 deletions

17
tensorflow/lite/micro/benchmarks/BUILD
View File

@ -51,3 +51,20 @@ cc_binary(
"//tensorflow/lite/micro/testing:micro_benchmark",
],
)
cc_binary(
name = "person_detection_benchmark",
srcs = ["person_detection_benchmark.cc"],
deps = [
"//tensorflow/lite:schema_fbs_version",
"//tensorflow/lite/c:common",
"//tensorflow/lite/micro:micro_framework",
"//tensorflow/lite/micro:micro_utils",
"//tensorflow/lite/micro/examples/person_detection:model_settings",
"//tensorflow/lite/micro/examples/person_detection:person_detect_model_data",
"//tensorflow/lite/micro/examples/person_detection:simple_images_test_data",
"//tensorflow/lite/micro/kernels:micro_ops",
"//tensorflow/lite/micro/testing:micro_benchmark",
"//tensorflow/lite/schema:schema_fbs",
],
)

14
tensorflow/lite/micro/benchmarks/Makefile.inc
View File

@ -1,3 +1,5 @@
$(eval $(call add_third_party_download,$(PERSON_MODEL_URL),$(PERSON_MODEL_MD5),person_model_grayscale,))
KEYWORD_BENCHMARK_SRCS := \
tensorflow/lite/micro/benchmarks/keyword_benchmark.cc \
tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.cc
@ -5,6 +7,18 @@ tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.cc
KEYWORD_BENCHMARK_HDRS := \
tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h
PERSON_DETECTION_BENCHMARK_SRCS := \
tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc \
$(MAKEFILE_DIR)/downloads/person_model_grayscale/no_person_image_data.cc \
$(MAKEFILE_DIR)/downloads/person_model_grayscale/person_detect_model_data.cc \
$(MAKEFILE_DIR)/downloads/person_model_grayscale/person_image_data.cc
PERSON_DETECTION_BENCHMARK_HDRS := \
tensorflow/lite/micro/examples/person_detection/person_detect_model_data.h
# Builds a standalone binary.
$(eval $(call microlite_test,keyword_benchmark,\
$(KEYWORD_BENCHMARK_SRCS),$(KEYWORD_BENCHMARK_HDRS)))
$(eval $(call microlite_test,person_detection_benchmark,\
$(PERSON_DETECTION_BENCHMARK_SRCS),$(PERSON_DETECTION_BENCHMARK_HDRS)))

33
tensorflow/lite/micro/benchmarks/README.md
View File

@ -7,8 +7,10 @@ platform.
## Table of contents
- [Keyword Benchmark](#keyword-benchmark)
- [Person Detection Benchmark](#person-detection-benchmark)
- [Run on x86](#run-on-x86)
- [Run on Xtensa XPG Simulator](#run-on-xtensa-xpg-simulator)
- [Run on Sparkfun Edge](#run-on-sparkfun-edge)
## Keyword benchmark
@ -17,17 +19,48 @@ weights and biases. This model is meant to test performance on a platform only.
Since the weights are scrambled, the output is meaningless. In order to validate
the accuracy of optimized kernels, please run the kernel tests.
## Person detection benchmark
The keyword benchmark provides a way to evaluate the performance of the 250KB
visual wakewords model.
## Run on x86
To run the keyword benchmark on x86, run
```
make -f tensorflow/lite/micro/tools/make/Makefile TAGS=posix test_keyword_benchmark
```
To run the person detection benchmark on x86, run
```
make -f tensorflow/lite/micro/tools/make/Makefile TAGS=posix test_person_detection_benchmark
```
## Run on Xtensa XPG Simulator
To run the keyword benchmark on the Xtensa XPG simulator, you will need a valid
Xtensa toolchain and license. With these set up, run:
```
make -f tensorflow/lite/micro/tools/make/Makefile TARGET=xtensa-xpg XTENSA_CORE=<xtensa core> TAGS=xtensa_hifimini test_keyword_benchmark -j18
```
## Run on Sparkfun Edge
The following instructions will help you build and deploy this benchmark on the
[SparkFun Edge development board](https://sparkfun.com/products/15170).
If you're new to using this board, we recommend walking through the
[AI on a microcontroller with TensorFlow Lite and SparkFun Edge](https://codelabs.developers.google.com/codelabs/sparkfun-tensorflow)
codelab to get an understanding of the workflow.
Build binary using
```
make -f tensorflow/lite/micro/tools/make/Makefile TARGET=sparkfun_edge person_detection_benchmark_bin
```
Refer to flashing instructions in the [Person Detection Example](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/micro/examples/person_detection/README.md#running-on-sparkfun-edge).

132
tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc Normal file
View File

@ -0,0 +1,132 @@
/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/lite/c/common.h"
#include "tensorflow/lite/micro/examples/person_detection/model_settings.h"
#include "tensorflow/lite/micro/examples/person_detection/no_person_image_data.h"
#include "tensorflow/lite/micro/examples/person_detection/person_detect_model_data.h"
#include "tensorflow/lite/micro/examples/person_detection/person_image_data.h"
#include "tensorflow/lite/micro/kernels/micro_ops.h"
#include "tensorflow/lite/micro/micro_error_reporter.h"
#include "tensorflow/lite/micro/micro_interpreter.h"
#include "tensorflow/lite/micro/micro_mutable_op_resolver.h"
#include "tensorflow/lite/micro/micro_utils.h"
#include "tensorflow/lite/micro/testing/micro_benchmark.h"
#include "tensorflow/lite/schema/schema_generated.h"
#include "tensorflow/lite/version.h"
// Create an area of memory to use for input, output, and intermediate arrays.
constexpr int tensor_arena_size = 73 * 1024;
uint8_t tensor_arena[tensor_arena_size];
/*
* Person Detection benchmark. Evaluates runtime performance of the visual
* wakewords person detection model. This is the same model found in
* exmaples/person_detection.
*/
namespace {
// Create an area of memory to use for input, output, and intermediate arrays.
constexpr int tensor_arena_size = 73 * 1024;
uint8_t tensor_arena[tensor_arena_size];
class PersonDetectionRunner {
public:
PersonDetectionRunner()
: person_detection_model_(tflite::GetModel(g_person_detect_model_data)),
reporter_(&micro_reporter_),
interpreter_(person_detection_model_, resolver_, tensor_arena,
tensor_arena_size, reporter_) {
resolver_.AddBuiltin(tflite::BuiltinOperator_DEPTHWISE_CONV_2D,
tflite::ops::micro::Register_DEPTHWISE_CONV_2D());
resolver_.AddBuiltin(tflite::BuiltinOperator_CONV_2D,
tflite::ops::micro::Register_CONV_2D());
resolver_.AddBuiltin(tflite::BuiltinOperator_AVERAGE_POOL_2D,
tflite::ops::micro::Register_AVERAGE_POOL_2D());
interpreter_.AllocateTensors();
TfLiteTensor* input = interpreter_.input(0);
TFLITE_CHECK_EQ(input->type, kTfLiteUInt8);
}
void RunSingleIterationWithPerson() {
// Populate input tensor with an image with a person
TfLiteTensor* input = interpreter_.input(0);
int8_t* input_buffer = tflite::GetTensorData<int8_t>(input);
int input_length = tflite::ElementCount(*input->dims);
for (int i = 0; i < input_length; i++) {
input_buffer[i] = g_person_data[i];
}
// Run the model on this input and make sure it succeeds.
TfLiteStatus invoke_status = interpreter_.Invoke();
if (invoke_status != kTfLiteOk) {
TF_LITE_REPORT_ERROR(reporter_, "Invoke failed.");
}
}
void RunSingleIterationWithoutPerson() {
// Populate input tensor with an image with no person.
TfLiteTensor* input = interpreter_.input(0);
int8_t* input_buffer = tflite::GetTensorData<int8_t>(input);
int input_length = tflite::ElementCount(*input->dims);
for (int i = 0; i < input_length; i++) {
input_buffer[i] = g_no_person_data[i];
}
// Run the model on this input and make sure it succeeds.
TfLiteStatus invoke_status = interpreter_.Invoke();
if (invoke_status != kTfLiteOk) {
TF_LITE_REPORT_ERROR(reporter_, "Invoke failed.");
}
}
private:
const tflite::Model* person_detection_model_;
tflite::MicroErrorReporter micro_reporter_;
tflite::ErrorReporter* reporter_;
tflite::MicroOpResolver<6> resolver_;
tflite::MicroInterpreter interpreter_;
};
// NOLINTNEXTLINE
PersonDetectionRunner runner;
void PersonDetectionFirstIteration() { runner.RunSingleIterationWithPerson(); }
void PersonDetectionTenIerationsWithPerson() {
// TODO(b/152644476): Add a way to run more than a single deterministic input.
for (int i = 0; i < 10; i++) {
runner.RunSingleIterationWithPerson();
}
}
void PersonDetectionTenIerationsWithoutPerson() {
// TODO(b/152644476): Add a way to run more than a single deterministic input.
for (int i = 0; i < 10; i++) {
runner.RunSingleIterationWithoutPerson();
}
}
} // namespace
TF_LITE_MICRO_BENCHMARKS_BEGIN
TF_LITE_MICRO_BENCHMARK(PersonDetectionFirstIteration);
TF_LITE_MICRO_BENCHMARK(PersonDetectionTenIerationsWithPerson);
TF_LITE_MICRO_BENCHMARK(PersonDetectionTenIerationsWithoutPerson);
TF_LITE_MICRO_BENCHMARKS_END

12
tensorflow/lite/micro/testing/micro_benchmark.h
View File

@ -16,6 +16,8 @@ limitations under the License.
#ifndef TENSORFLOW_LITE_MICRO_TESTING_MICRO_BENCHMARK_H_
#define TENSORFLOW_LITE_MICRO_TESTING_MICRO_BENCHMARK_H_
#include <climits>
#include "tensorflow/lite/micro/micro_error_reporter.h"
#include "tensorflow/lite/micro/micro_time.h"
@ -35,7 +37,9 @@ extern tflite::ErrorReporter* reporter;
int32_t duration_ticks; \
int32_t duration_ms;
#define TF_LITE_MICRO_BENCHMARKS_END }
#define TF_LITE_MICRO_BENCHMARKS_END \
return 0; \
}
#define TF_LITE_MICRO_BENCHMARK(func) \
if (tflite::ticks_per_second() == 0) { \
@ -44,7 +48,11 @@ extern tflite::ErrorReporter* reporter;
start_ticks = tflite::GetCurrentTimeTicks(); \
func(); \
duration_ticks = tflite::GetCurrentTimeTicks() - start_ticks; \
duration_ms = (duration_ticks * 1000) / tflite::ticks_per_second(); \
if (duration_ticks > INT_MAX / 1000) { \
duration_ms = duration_ticks / (tflite::ticks_per_second() / 1000); \
} else { \
duration_ms = (duration_ticks * 1000) / tflite::ticks_per_second(); \
} \
TF_LITE_REPORT_ERROR(micro_benchmark::reporter, "%s took %d ticks (%d ms)", \
#func, duration_ticks, duration_ms);

4
tensorflow/lite/micro/tools/make/helper_functions.inc
View File

@ -384,7 +384,9 @@ $(call generate_project,make,$(MAKE_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(T
$(call generate_arc_project,make,$(MAKE_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(LDFLAGS) $(GENERATED_PROJECT_LIBS),$(CXXFLAGS) $(GENERATED_PROJECT_INCLUDES), $(CCFLAGS) $(GENERATED_PROJECT_INCLUDES))
$(call generate_project,mbed,$(MBED_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS),$(CCFLAGS),$(TARGET_TOOLCHAIN_ROOT),$(TARGET_TOOLCHAIN_PREFIX))
$(call generate_project,keil,$(KEIL_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS),$(CCFLAGS),$(TARGET_TOOLCHAIN_ROOT),$(TARGET_TOOLCHAIN_PREFIX))
$(call generate_arduino_project,$(ARDUINO_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS),$(CCFLAGS))
ifeq (,$(findstring _benchmark,$(1)))
$(call generate_arduino_project,$(ARDUINO_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS) $(2),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS) $(3),$(MICROLITE_LIBS),$(CXXFLAGS),$(CCFLAGS))
endif
$(call generate_esp_project,$(ESP_PROJECT_FILES),$(1),$(MICROLITE_CC_SRCS) $(THIRD_PARTY_CC_SRCS),$(MICROLITE_CC_HDRS) $(THIRD_PARTY_CC_HDRS) $(MICROLITE_TEST_HDRS),$(2),$(3),$(MICROLITE_LIBS),$(CXXFLAGS),$(CCFLAGS),$(PROJECT_INCLUDES))
endef