From b0092ab899cdeff58e61164269b5e0be1b86f433 Mon Sep 17 00:00:00 2001 From: Nat Jeffries Date: Wed, 2 Dec 2020 16:21:48 -0800 Subject: [PATCH] Move person_detection_experimental to person_detection and scrap uint8 version. Addresses https://github.com/tensorflow/tensorflow/issues/44912 PiperOrigin-RevId: 345335944 Change-Id: If46bdf590faf66d4158596030b2f41a50c9eb388 --- tensorflow/lite/micro/benchmarks/BUILD | 18 - tensorflow/lite/micro/benchmarks/Makefile.inc | 16 +- .../benchmarks/person_detection_benchmark.cc | 30 +- ...person_detection_experimental_benchmark.cc | 92 --- .../micro/examples/person_detection/BUILD | 6 - .../examples/person_detection/Makefile.inc | 16 +- .../micro/examples/person_detection/README.md | 137 +++- .../apollo3evb/image_provider.cc | 2 - .../person_detection/arc_emsdp/Makefile.inc | 45 +- .../arc_emsdp/emsdp.lcf | 0 .../arduino/detection_responder.cc | 2 +- .../arduino/image_provider.cc | 9 +- .../person_detection/detection_responder.cc | 2 +- .../person_detection/detection_responder.h | 2 +- .../detection_responder_test.cc | 4 +- .../himax_driver/HM01B0_optimized.c | 9 +- .../himax_driver/HM01B0_optimized.h | 2 +- .../himax_we1_evb/detection_responder.cc | 12 +- .../himax_we1_evb/image_provider.cc | 14 +- .../person_detection/image_provider.cc | 2 +- .../person_detection/image_provider.h | 2 +- .../person_detection/image_provider_test.cc | 2 +- .../person_detection/main_functions.cc | 20 +- .../person_detection/model_settings.cc | 3 +- .../person_detection/model_settings.h | 4 +- .../person_detection/person_detection_test.cc | 48 +- .../sparkfun_edge/detection_responder.cc | 2 +- .../sparkfun_edge/image_provider.cc | 2 +- .../person_detection/training_a_model.md | 3 + .../person_detection_experimental/BUILD | 138 ---- .../Makefile.inc | 83 -- .../person_detection_experimental/README.md | 570 -------------- .../apollo3evb/image_provider.cc | 198 ----- .../arc_emsdp/Makefile.inc | 43 -- .../arduino/detection_responder.cc | 56 -- .../arduino/image_provider.cc | 266 ------- .../arduino/main.cc | 20 - .../detection_responder.cc | 25 - .../detection_responder.h | 34 - .../detection_responder_test.cc | 32 - .../himax_driver/HM01B0.c | 719 ------------------ .../himax_driver/HM01B0.h | 402 ---------- .../HM01B0_RAW8_QVGA_8bits_lsb_5fps.h | 510 ------------- .../himax_driver/HM01B0_Walking1s_01.h | 56 -- .../himax_driver/HM01B0_Walking1s_01.txt | 8 - .../himax_driver/HM01B0_debug.c | 35 - .../himax_driver/HM01B0_debug.h | 49 -- .../himax_driver/HM01B0_optimized.c | 87 --- .../himax_driver/HM01B0_optimized.h | 50 -- .../himax_driver/Makefile.inc | 13 - .../himax_driver/platform_Sparkfun_Edge.h | 54 -- .../image_provider.cc | 26 - .../image_provider.h | 39 - .../image_provider_test.cc | 43 -- .../person_detection_experimental/main.cc | 27 - .../main_functions.cc | 117 --- .../main_functions.h | 28 - .../model_settings.cc | 21 - .../model_settings.h | 35 - .../no_person_image_data.h | 30 - .../person_detect_model_data.h | 27 - .../person_detection_test.cc | 135 ---- .../person_image_data.h | 30 - .../sparkfun_edge/detection_responder.cc | 54 -- .../sparkfun_edge/image_provider.cc | 203 ----- .../training_a_model.md | 455 ----------- .../lite/micro/kernels/arc_mli/README.md | 2 +- .../tools/make/fix_arduino_subfolders.py | 25 +- .../tools/make/fix_arduino_subfolders_test.sh | 2 +- .../tools/make/targets/stm32f4_makefile.inc | 2 +- .../tools/make/targets/xtensa_makefile.inc | 3 +- .../tools/make/third_party_downloads.inc | 4 +- 72 files changed, 269 insertions(+), 4993 deletions(-) delete mode 100644 tensorflow/lite/micro/benchmarks/person_detection_experimental_benchmark.cc rename tensorflow/lite/micro/examples/{person_detection_experimental => person_detection}/arc_emsdp/emsdp.lcf (100%) rename tensorflow/lite/micro/examples/{person_detection_experimental => person_detection}/himax_we1_evb/detection_responder.cc (83%) rename tensorflow/lite/micro/examples/{person_detection_experimental => person_detection}/himax_we1_evb/image_provider.cc (80%) delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/BUILD delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/Makefile.inc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/README.md delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/apollo3evb/image_provider.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/arc_emsdp/Makefile.inc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/arduino/detection_responder.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/arduino/image_provider.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/arduino/main.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/detection_responder_test.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.c delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_RAW8_QVGA_8bits_lsb_5fps.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_Walking1s_01.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_Walking1s_01.txt delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.c delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_optimized.c delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_optimized.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/Makefile.inc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/platform_Sparkfun_Edge.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/image_provider.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/image_provider.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/image_provider_test.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/main.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/main_functions.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/model_settings.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/no_person_image_data.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/person_image_data.h delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/sparkfun_edge/detection_responder.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/sparkfun_edge/image_provider.cc delete mode 100644 tensorflow/lite/micro/examples/person_detection_experimental/training_a_model.md diff --git a/tensorflow/lite/micro/benchmarks/BUILD b/tensorflow/lite/micro/benchmarks/BUILD index f2eb0144d32..c37947aefee 100644 --- a/tensorflow/lite/micro/benchmarks/BUILD +++ b/tensorflow/lite/micro/benchmarks/BUILD @@ -62,21 +62,3 @@ cc_binary( "//tensorflow/lite/schema:schema_fbs", ], ) - -cc_binary( - name = "person_detection_experimental_benchmark", - srcs = ["person_detection_experimental_benchmark.cc"], - deps = [ - ":micro_benchmark", - "//tensorflow/lite:version", - "//tensorflow/lite/c:common", - "//tensorflow/lite/micro:micro_error_reporter", - "//tensorflow/lite/micro:micro_framework", - "//tensorflow/lite/micro:micro_utils", - "//tensorflow/lite/micro:op_resolvers", - "//tensorflow/lite/micro/examples/person_detection_experimental:model_settings", - "//tensorflow/lite/micro/examples/person_detection_experimental:person_detect_model_data", - "//tensorflow/lite/micro/examples/person_detection_experimental:simple_images_test_data", - "//tensorflow/lite/schema:schema_fbs", - ], -) diff --git a/tensorflow/lite/micro/benchmarks/Makefile.inc b/tensorflow/lite/micro/benchmarks/Makefile.inc index a47bc2e723a..998851bd1ce 100644 --- a/tensorflow/lite/micro/benchmarks/Makefile.inc +++ b/tensorflow/lite/micro/benchmarks/Makefile.inc @@ -7,21 +7,12 @@ tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h PERSON_DETECTION_BENCHMARK_SRCS := \ tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc \ -$(MAKEFILE_DIR)/downloads/person_model_grayscale/no_person_image_data.cc \ -$(MAKEFILE_DIR)/downloads/person_model_grayscale/person_detect_model_data.cc \ -$(MAKEFILE_DIR)/downloads/person_model_grayscale/person_image_data.cc - -PERSON_DETECTION_BENCHMARK_HDRS := \ -tensorflow/lite/micro/examples/person_detection/person_detect_model_data.h - -PERSON_DETECTION_EXPERIMENTAL_BENCHMARK_SRCS := \ -tensorflow/lite/micro/benchmarks/person_detection_experimental_benchmark.cc \ $(MAKEFILE_DIR)/downloads/person_model_int8/no_person_image_data.cc \ $(MAKEFILE_DIR)/downloads/person_model_int8/person_detect_model_data.cc \ $(MAKEFILE_DIR)/downloads/person_model_int8/person_image_data.cc -PERSON_DETECTION_EXPERIMENTAL_BENCHMARK_HDRS := \ -tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h +PERSON_DETECTION_BENCHMARK_HDRS := \ +tensorflow/lite/micro/examples/person_detection/person_detect_model_data.h # Builds a standalone binary. $(eval $(call microlite_test,keyword_benchmark,\ @@ -30,6 +21,3 @@ $(KEYWORD_BENCHMARK_SRCS),$(KEYWORD_BENCHMARK_HDRS))) $(eval $(call microlite_test,person_detection_benchmark,\ $(PERSON_DETECTION_BENCHMARK_SRCS),$(PERSON_DETECTION_BENCHMARK_HDRS))) -$(eval $(call microlite_test,person_detection_experimental_benchmark,\ -$(PERSON_DETECTION_EXPERIMENTAL_BENCHMARK_SRCS),$(PERSON_DETECTION_EXPERIMENTAL_BENCHMARK_HDRS))) - diff --git a/tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc b/tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc index 687440ac9fb..7b38708e91c 100644 --- a/tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc +++ b/tensorflow/lite/micro/benchmarks/person_detection_benchmark.cc @@ -35,13 +35,11 @@ limitations under the License. namespace { using PersonDetectionOpResolver = tflite::AllOpsResolver; -using PersonDetectionBenchmarkRunner = MicroBenchmarkRunner; - -constexpr int kRandomSeed = 42; +using PersonDetectionBenchmarkRunner = MicroBenchmarkRunner; // Create an area of memory to use for input, output, and intermediate arrays. // Align arena to 16 bytes to avoid alignment warnings on certain platforms. -constexpr int kTensorArenaSize = 95 * 1024; +constexpr int kTensorArenaSize = 135 * 1024; alignas(16) uint8_t tensor_arena[kTensorArenaSize]; uint8_t op_resolver_buffer[sizeof(PersonDetectionOpResolver)]; @@ -52,9 +50,9 @@ PersonDetectionBenchmarkRunner* benchmark_runner = nullptr; // issues on Sparkfun. Use new since static variables within a method // are automatically surrounded by locking, which breaks bluepill and stm32f4. void CreateBenchmarkRunner() { - // We allocate PersonDetectionOpResolver from a global buffer because the - // object's lifetime must exceed that of the PersonDetectionBenchmarkRunner - // object. + // We allocate PersonDetectionOpResolver from a global buffer + // because the object's lifetime must exceed that of the + // PersonDetectionBenchmarkRunner object. benchmark_runner = new (benchmark_runner_buffer) PersonDetectionBenchmarkRunner(g_person_detect_model_data, new (op_resolver_buffer) @@ -62,24 +60,20 @@ void CreateBenchmarkRunner() { tensor_arena, kTensorArenaSize); } -void PersonDetectionTenIterationsWithRandomInput() { - benchmark_runner->SetRandomInput(kRandomSeed); - for (int i = 0; i < 10; i++) { - benchmark_runner->RunSingleIteration(); - } +void InitializeBenchmarkRunner() { + CreateBenchmarkRunner(); + benchmark_runner->SetInput(reinterpret_cast(g_person_data)); } void PersonDetectionTenIerationsWithPerson() { - // TODO(b/152644476): Add a way to run more than a single deterministic input. - benchmark_runner->SetInput(g_person_data); + benchmark_runner->SetInput(reinterpret_cast(g_person_data)); for (int i = 0; i < 10; i++) { benchmark_runner->RunSingleIteration(); } } void PersonDetectionTenIerationsWithoutPerson() { - // TODO(b/152644476): Add a way to run more than a single deterministic input. - benchmark_runner->SetInput(g_no_person_data); + benchmark_runner->SetInput(reinterpret_cast(g_no_person_data)); for (int i = 0; i < 10; i++) { benchmark_runner->RunSingleIteration(); } @@ -89,8 +83,8 @@ void PersonDetectionTenIerationsWithoutPerson() { TF_LITE_MICRO_BENCHMARKS_BEGIN -TF_LITE_MICRO_BENCHMARK(CreateBenchmarkRunner()); -TF_LITE_MICRO_BENCHMARK(PersonDetectionTenIterationsWithRandomInput()); +TF_LITE_MICRO_BENCHMARK(InitializeBenchmarkRunner()); +TF_LITE_MICRO_BENCHMARK(benchmark_runner->RunSingleIteration()); TF_LITE_MICRO_BENCHMARK(PersonDetectionTenIerationsWithPerson()); TF_LITE_MICRO_BENCHMARK(PersonDetectionTenIerationsWithoutPerson()); diff --git a/tensorflow/lite/micro/benchmarks/person_detection_experimental_benchmark.cc b/tensorflow/lite/micro/benchmarks/person_detection_experimental_benchmark.cc deleted file mode 100644 index 68850c90188..00000000000 --- a/tensorflow/lite/micro/benchmarks/person_detection_experimental_benchmark.cc +++ /dev/null @@ -1,92 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/micro/all_ops_resolver.h" -#include "tensorflow/lite/micro/benchmarks/micro_benchmark.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/no_person_image_data.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/person_image_data.h" -#include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/micro_interpreter.h" -#include "tensorflow/lite/micro/micro_utils.h" -#include "tensorflow/lite/schema/schema_generated.h" -#include "tensorflow/lite/version.h" - -/* - * Person Detection benchmark. Evaluates runtime performance of the visual - * wakewords person detection model. This is the same model found in - * exmaples/person_detection. - */ - -namespace { - -using PersonDetectionExperimentalOpResolver = tflite::AllOpsResolver; -using PersonDetectionExperimentalBenchmarkRunner = MicroBenchmarkRunner; - -// Create an area of memory to use for input, output, and intermediate arrays. -// Align arena to 16 bytes to avoid alignment warnings on certain platforms. -constexpr int kTensorArenaSize = 135 * 1024; -alignas(16) uint8_t tensor_arena[kTensorArenaSize]; - -uint8_t op_resolver_buffer[sizeof(PersonDetectionExperimentalOpResolver)]; -uint8_t - benchmark_runner_buffer[sizeof(PersonDetectionExperimentalBenchmarkRunner)]; -PersonDetectionExperimentalBenchmarkRunner* benchmark_runner = nullptr; - -// Initialize benchmark runner instance explicitly to avoid global init order -// issues on Sparkfun. Use new since static variables within a method -// are automatically surrounded by locking, which breaks bluepill and stm32f4. -void CreateBenchmarkRunner() { - // We allocate PersonDetectionExperimentalOpResolver from a global buffer - // because the object's lifetime must exceed that of the - // PersonDetectionBenchmarkRunner object. - benchmark_runner = - new (benchmark_runner_buffer) PersonDetectionExperimentalBenchmarkRunner( - g_person_detect_model_data, - new (op_resolver_buffer) PersonDetectionExperimentalOpResolver(), - tensor_arena, kTensorArenaSize); -} - -void InitializeBenchmarkRunner() { - CreateBenchmarkRunner(); - benchmark_runner->SetInput(reinterpret_cast(g_person_data)); -} - -void PersonDetectionTenIerationsWithPerson() { - benchmark_runner->SetInput(reinterpret_cast(g_person_data)); - for (int i = 0; i < 10; i++) { - benchmark_runner->RunSingleIteration(); - } -} - -void PersonDetectionTenIerationsWithoutPerson() { - benchmark_runner->SetInput(reinterpret_cast(g_no_person_data)); - for (int i = 0; i < 10; i++) { - benchmark_runner->RunSingleIteration(); - } -} - -} // namespace - -TF_LITE_MICRO_BENCHMARKS_BEGIN - -TF_LITE_MICRO_BENCHMARK(InitializeBenchmarkRunner()); -TF_LITE_MICRO_BENCHMARK(benchmark_runner->RunSingleIteration()); -TF_LITE_MICRO_BENCHMARK(PersonDetectionTenIerationsWithPerson()); -TF_LITE_MICRO_BENCHMARK(PersonDetectionTenIerationsWithoutPerson()); - -TF_LITE_MICRO_BENCHMARKS_END diff --git a/tensorflow/lite/micro/examples/person_detection/BUILD b/tensorflow/lite/micro/examples/person_detection/BUILD index e9b0df65c48..7e0611d6a59 100644 --- a/tensorflow/lite/micro/examples/person_detection/BUILD +++ b/tensorflow/lite/micro/examples/person_detection/BUILD @@ -135,9 +135,3 @@ cc_binary( "//tensorflow/lite/schema:schema_fbs", ], ) - -sh_test( - name = "person_detection_binary_test", - srcs = ["person_detection_binary_test.sh"], - data = [":person_detection"], -) diff --git a/tensorflow/lite/micro/examples/person_detection/Makefile.inc b/tensorflow/lite/micro/examples/person_detection/Makefile.inc index 8dea8f96549..6b8be54cb07 100644 --- a/tensorflow/lite/micro/examples/person_detection/Makefile.inc +++ b/tensorflow/lite/micro/examples/person_detection/Makefile.inc @@ -1,6 +1,8 @@ +$(eval $(call add_third_party_download,$(PERSON_MODEL_INT8_URL),$(PERSON_MODEL_INT8_MD5),person_model_int8,)) + person_detection_MODEL_SRCS := \ tensorflow/lite/micro/examples/person_detection/model_settings.cc \ -$(MAKEFILE_DIR)/downloads/person_model_grayscale/person_detect_model_data.cc +$(MAKEFILE_DIR)/downloads/person_model_int8/person_detect_model_data.cc person_detection_MODEL_HDRS := \ tensorflow/lite/micro/examples/person_detection/model_settings.h \ @@ -8,8 +10,8 @@ tensorflow/lite/micro/examples/person_detection/person_detect_model_data.h person_detection_TEST_SRCS := \ tensorflow/lite/micro/examples/person_detection/person_detection_test.cc \ -$(MAKEFILE_DIR)/downloads/person_model_grayscale/no_person_image_data.cc \ -$(MAKEFILE_DIR)/downloads/person_model_grayscale/person_image_data.cc \ +$(MAKEFILE_DIR)/downloads/person_model_int8/no_person_image_data.cc \ +$(MAKEFILE_DIR)/downloads/person_model_int8/person_image_data.cc \ $(person_detection_MODEL_SRCS) person_detection_TEST_HDRS := \ @@ -50,7 +52,7 @@ $(person_detection_MODEL_HDRS) include $(wildcard tensorflow/lite/micro/examples/person_detection/*/Makefile.inc) # Tests loading and running a vision model. -$(eval $(call microlite_test,person_detection_test,\ +$(eval $(call microlite_test,person_detection_test_int8,\ $(person_detection_TEST_SRCS),$(person_detection_TEST_HDRS))) # Three conflicting issues here: @@ -68,14 +70,14 @@ $(person_detection_TEST_SRCS),$(person_detection_TEST_HDRS))) # basically equivalent). ifneq ($(TARGET),sparkfun_edge) # Tests the image provider module. -$(eval $(call microlite_test,image_provider_test,\ +$(eval $(call microlite_test,image_provider_test_int8,\ $(IMAGE_PROVIDER_TEST_SRCS),$(IMAGE_PROVIDER_TEST_HDRS))) endif # Tests the detection responder module. -$(eval $(call microlite_test,detection_responder_test,\ +$(eval $(call microlite_test,detection_responder_test_int8,\ $(DETECTION_RESPONDER_TEST_SRCS),$(DETECTION_RESPONDER_TEST_HDRS))) # Builds a standalone object recognition binary. -$(eval $(call microlite_test,person_detection,\ +$(eval $(call microlite_test,person_detection_int8,\ $(person_detection_SRCS),$(person_detection_HDRS))) diff --git a/tensorflow/lite/micro/examples/person_detection/README.md b/tensorflow/lite/micro/examples/person_detection/README.md index 7312582f9b9..1bd2a5ddf2c 100644 --- a/tensorflow/lite/micro/examples/person_detection/README.md +++ b/tensorflow/lite/micro/examples/person_detection/README.md @@ -1,10 +1,9 @@ - - # Person detection example This example shows how you can use Tensorflow Lite to run a 250 kilobyte neural network to recognize people in images captured by a camera. It is designed to run on systems with small amounts of memory such as microcontrollers and DSPs. +This uses the experimental int8 quantized version of the person detection model. ## Table of contents @@ -12,6 +11,7 @@ run on systems with small amounts of memory such as microcontrollers and DSPs. - [Running on ARC EM SDP](#running-on-arc-em-sdp) - [Running on Arduino](#running-on-arduino) - [Running on ESP32](#running-on-esp32) +- [Running on HIMAX WE1 EVB](#running-on-himax-we1-evb) - [Running on SparkFun Edge](#running-on-sparkfun-edge) - [Run the tests on a development machine](#run-the-tests-on-a-development-machine) - [Debugging image capture](#debugging-image-capture) @@ -25,10 +25,8 @@ board. General information and instructions on using the board with TensorFlow Lite Micro can be found in the common [ARC targets description](/tensorflow/lite/micro/tools/make/targets/arc/README.md). -This example is quantized with symmetric uint8 scheme. As noted in -[kernels/arc_mli/README.md](/tensorflow/lite/micro/kernels/arc_mli/README.md), -embARC MLI supports optimized kernels for int8 quantization only. Therefore, -this example will only use TFLM reference kernels. +This example uses asymmetric int8 quantization and can therefore leverage +optimized int8 kernels from the embARC MLI library The ARC EM SDP board contains a rich set of extension interfaces. You can choose any compatible camera and modify @@ -53,9 +51,14 @@ The example project for ARC EM SDP platform can be generated with the following command: ``` -make -f tensorflow/lite/micro/tools/make/Makefile TARGET=arc_emsdp TAGS=no_arc_mli generate_person_detection_make_project +make -f tensorflow/lite/micro/tools/make/Makefile \ +TARGET=arc_emsdp TAGS=reduce_codesize \ +generate_person_detection_int8_make_project ``` +Note that `TAGS=reduce_codesize` applies example specific changes of code to +reduce total size of application. It can be omitted. + ### Build and Run Example For more detailed information on building and running examples see the @@ -73,7 +76,7 @@ get it started. 2. Go to the generated example project director ``` - cd tensorflow/lite/micro/tools/make/gen/arc_emsdp_arc/prj/person_detection/make + cd tensorflow/lite/micro/tools/make/gen/arc_emsdp_arc/prj/person_detection_int8/make ``` 3. Build the example using @@ -141,7 +144,7 @@ Connect the Arducam pins as follows: ### Install the Arduino_TensorFlowLite library Download the current nightly build of the library: -[person_detection.zip](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/tensorflow/lite/micro/tools/make/gen/arduino_x86_64/prj/person_detection/tensorflow_lite.zip) +[person_detection.zip](https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_int8_grayscale_2020_01_13.zip) This example application is included as part of the official TensorFlow Lite Arduino library. To install it, open the Arduino library manager in @@ -344,6 +347,112 @@ The previous two commands can be combined: idf.py --port /dev/ttyUSB0 flash monitor ``` +## Running on HIMAX WE1 EVB + +The following instructions will help you build and deploy this example to +[HIMAX WE1 EVB](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_board_brief) +board. To understand more about using this board, please check +[HIMAX WE1 EVB user guide](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide). + +### Initial Setup + +To use the HIMAX WE1 EVB, please make sure following software are installed: + +#### MetaWare Development Toolkit + +See +[Install the Synopsys DesignWare ARC MetaWare Development Toolkit](/tensorflow/lite/micro/tools/make/targets/arc/README.md#install-the-synopsys-designware-arc-metaware-development-toolkit) +section for instructions on toolchain installation. + +#### Make Tool version + +A `'make'` tool is required for deploying Tensorflow Lite Micro applications on +HIMAX WE1 EVB, See +[Check make tool version](/tensorflow/lite/micro/tools/make/targets/arc/README.md#make-tool) +section for proper environment. + +#### Serial Terminal Emulation Application + +There are 2 main purposes for HIMAX WE1 EVB Debug UART port + +- print application output +- burn application to flash by using xmodem send application binary + +You can use any terminal emulation program (like [PuTTY](https://www.putty.org/) +or [minicom](https://linux.die.net/man/1/minicom)). + +### Generate Example Project + +The example project for HIMAX WE1 EVB platform can be generated with the +following command: + +Download related third party data + +``` +make -f tensorflow/lite/micro/tools/make/Makefile TARGET=himax_we1_evb third_party_downloads +``` + +Generate person detection project + +``` +make -f tensorflow/lite/micro/tools/make/Makefile generate_person_detection_int8_make_project TARGET=himax_we1_evb +``` + +### Build and Burn Example + +Following the Steps to run person detection example at HIMAX WE1 EVB platform. + +1. Go to the generated example project directory. + + ``` + cd tensorflow/lite/micro/tools/make/gen/himax_we1_evb_arc/prj/person_detection_int8/make + ``` + +2. Build the example using + + ``` + make app + ``` + +3. After example build finish, copy ELF file and map file to image generate + tool directory. \ + image generate tool directory located at + `'tensorflow/lite/micro/tools/make/downloads/himax_we1_sdk/image_gen_linux_v3/'` + + ``` + cp person_detection_int8.elf himax_we1_evb.map ../../../../../downloads/himax_we1_sdk/image_gen_linux_v3/ + ``` + +4. Go to flash image generate tool directory. + + ``` + cd ../../../../../downloads/himax_we1_sdk/image_gen_linux_v3/ + ``` + + make sure this tool directory is in $PATH. You can permanently set it to + PATH by + + ``` + export PATH=$PATH:$(pwd) + ``` + +5. run image generate tool, generate flash image file. + + * Before running image generate tool, by typing `sudo chmod +x image_gen` + and `sudo chmod +x sign_tool` to make sure it is executable. + + ``` + image_gen -e person_detection_int8.elf -m himax_we1_evb.map -o out.img + ``` + +6. Download flash image file to HIMAX WE1 EVB by UART: + + * more detail about download image through UART can be found at + [HIMAX WE1 EVB update Flash image](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide#flash-image-update) + +After these steps, press reset button on the HIMAX WE1 EVB, you will see +application output in the serial terminal. + ## Running on SparkFun Edge The following instructions will help you build and deploy this sample on the @@ -381,14 +490,14 @@ Enter the following command to set up some dummy cryptographic keys we can use for development: ``` -cp tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.2.0/tools/apollo3_scripts/keys_info0.py \ -tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.2.0/tools/apollo3_scripts/keys_info.py +cp tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/keys_info0.py \ +tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/keys_info.py ``` Next, run the following command to create a signed binary: ``` -python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.2.0/tools/apollo3_scripts/create_cust_image_blob.py \ +python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/create_cust_image_blob.py \ --bin tensorflow/lite/micro/tools/make/gen/sparkfun_edge_cortex-m4/bin/person_detection.bin \ --load-address 0xC000 \ --magic-num 0xCB \ @@ -401,7 +510,7 @@ command to create a final version of the file that can be used to flash our device with the bootloader script we will use in the next step: ``` -python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.2.0/tools/apollo3_scripts/create_cust_wireupdate_blob.py \ +python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/create_cust_wireupdate_blob.py \ --load-address 0x20000 \ --bin main_nonsecure_ota.bin \ -i 6 \ @@ -437,7 +546,7 @@ hit the button marked `RST`. Continue holding the button marked `14` while running the following command: ``` -python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.2.0/tools/apollo3_scripts/uart_wired_update.py \ +python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/uart_wired_update.py \ -b ${BAUD_RATE} ${DEVICENAME} \ -r 1 \ -f main_nonsecure_wire.bin \ diff --git a/tensorflow/lite/micro/examples/person_detection/apollo3evb/image_provider.cc b/tensorflow/lite/micro/examples/person_detection/apollo3evb/image_provider.cc index f6ed6bcab66..1dd53e412f9 100644 --- a/tensorflow/lite/micro/examples/person_detection/apollo3evb/image_provider.cc +++ b/tensorflow/lite/micro/examples/person_detection/apollo3evb/image_provider.cc @@ -147,12 +147,10 @@ TfLiteStatus InitCamera(tflite::ErrorReporter* error_reporter) { hm01b0_power_up(&s_HM01B0Cfg); - // TODO(njeff): check the delay time to just fit the spec. am_util_delay_ms(1); hm01b0_mclk_enable(&s_HM01B0Cfg); - // TODO(njeff): check the delay time to just fit the spec. am_util_delay_ms(1); hm01b0_init_if(&s_HM01B0Cfg); diff --git a/tensorflow/lite/micro/examples/person_detection/arc_emsdp/Makefile.inc b/tensorflow/lite/micro/examples/person_detection/arc_emsdp/Makefile.inc index 29a09466e83..1555f78e9b2 100644 --- a/tensorflow/lite/micro/examples/person_detection/arc_emsdp/Makefile.inc +++ b/tensorflow/lite/micro/examples/person_detection/arc_emsdp/Makefile.inc @@ -1,24 +1,43 @@ ifeq ($(TARGET), arc_emsdp) -# Patch of arc make project to adjust it specifically -# for person detection example. In particular: +#Patch of arc make project to adjust it specifically +#for experimental person detection example.In particular: # - Use Linker command file with better usage of fast memory -# - In case project was generated with MLI usage, reduce scratch buffers. +#- Stripout TFLM reference code by default. +#- Optional : replace mli switchers with specialized kernels +#for smaller code size person_detection_HDRS += \ - person_detection_patch.txt + person_detection_int8_patch.txt person_detection_TEST_HDRS += \ - person_detection_patch.txt + person_detection_int8_patch.txt + ARC_MLI_BACKEND_PATH = /tensorflow/lite/micro/kernels/arc_mli -%/person_detection_patch.txt: %/emsdp.lcf %/Makefile - @cp tensorflow/lite/micro/tools/make/targets/arc/emsdp/emsdp_v2.lcf $< - @echo emsdp.lcf > $@ - @sed -E -i 's#MLI_ONLY *\?= *false#MLI_ONLY \?= false\n\ - CXXFLAGS += -DSCRATCH_MEM_X_SIZE=0 -DSCRATCH_MEM_Y_SIZE=0 -DSCRATCH_MEM_Z_SIZE=0\ - CCFLAGS += -DSCRATCH_MEM_X_SIZE=0 -DSCRATCH_MEM_Y_SIZE=0 -DSCRATCH_MEM_Z_SIZE=0#'\ - $(word 2, $^) - @echo Makefile >> $@ +#Apply changes in generated project files. +#See related comment echoed(@echo ) after each change +#to get understanding on it's purpose. +%/person_detection_int8_patch.txt: %/emsdp.lcf %/Makefile %$(ARC_MLI_BACKEND_PATH)/conv.cc %$(ARC_MLI_BACKEND_PATH)/depthwise_conv.cc %$(ARC_MLI_BACKEND_PATH)/pooling.cc + @cp tensorflow/lite/micro/examples/person_detection/arc_emsdp/emsdp.lcf $< + @echo emsdp.lcf: Replace with example specific memory map > $@ + + @sed -E -i 's#MLI_ONLY *\?= *false#MLI_ONLY \?= true#' $(word 2, $^) + @echo Makefile: No Reference fallback for MLI supported functions >> $@ + +ifneq ($(filter $(ALL_TAGS), reduce_codesize),) +#In case 'reduce_codesize' tag is present, we replace common MLI functions with +#specializations appropriate for this particular graph.But such changes of code +#with high probability may not be acceptable for other graphs and will need +#to be adjusted by the user + + @sed -E -i 's#mli_krn_conv2d_nhwc_sa8_sa8_sa32#mli_krn_conv2d_nhwc_sa8_sa8_sa32_k1x1_nopad#' $(word 3, $^) + @sed -E -i 's#mli_krn_depthwise_conv2d_hwcn_sa8_sa8_sa32#mli_krn_depthwise_conv2d_hwcn_sa8_sa8_sa32_k3x3_krnpad#' $(word 4, $^) + @sed -E -i 's#mli_krn_avepool_hwc_sa8#mli_krn_avepool_hwc_sa8_k3x3_nopad#' $(word 5, $^) + @sed -E -i 's#mli_krn_maxpool_hwc_sa8\(in_ptr, \&cfg, out_ptr\);#return kTfLiteError;#' $(word 5, $^) + @echo $(word 3, $^): Use specialization >> $@ + @echo $(word 4, $^): Use specialization >> $@ + @echo $(word 5, $^): Use specialization and remove max pooling >> $@ +endif endif diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/arc_emsdp/emsdp.lcf b/tensorflow/lite/micro/examples/person_detection/arc_emsdp/emsdp.lcf similarity index 100% rename from tensorflow/lite/micro/examples/person_detection_experimental/arc_emsdp/emsdp.lcf rename to tensorflow/lite/micro/examples/person_detection/arc_emsdp/emsdp.lcf diff --git a/tensorflow/lite/micro/examples/person_detection/arduino/detection_responder.cc b/tensorflow/lite/micro/examples/person_detection/arduino/detection_responder.cc index 622972092cb..672ab908cf8 100644 --- a/tensorflow/lite/micro/examples/person_detection/arduino/detection_responder.cc +++ b/tensorflow/lite/micro/examples/person_detection/arduino/detection_responder.cc @@ -25,7 +25,7 @@ limitations under the License. // Flash the blue LED after each inference void RespondToDetection(tflite::ErrorReporter* error_reporter, - uint8_t person_score, uint8_t no_person_score) { + int8_t person_score, int8_t no_person_score) { static bool is_initialized = false; if (!is_initialized) { // Pins for the built-in RGB LEDs on the Arduino Nano 33 BLE Sense diff --git a/tensorflow/lite/micro/examples/person_detection/arduino/image_provider.cc b/tensorflow/lite/micro/examples/person_detection/arduino/image_provider.cc index cd79e6b2d44..f1a3956bf40 100644 --- a/tensorflow/lite/micro/examples/person_detection/arduino/image_provider.cc +++ b/tensorflow/lite/micro/examples/person_detection/arduino/image_provider.cc @@ -150,7 +150,7 @@ TfLiteStatus ReadData(tflite::ErrorReporter* error_reporter) { // Decode the JPEG image, crop it, and convert it to greyscale TfLiteStatus DecodeAndProcessImage(tflite::ErrorReporter* error_reporter, int image_width, int image_height, - uint8_t* image_data) { + int8_t* image_data) { TF_LITE_REPORT_ERROR(error_reporter, "Decoding JPEG and converting to greyscale"); // Parse the JPEG headers. The image will be decoded as a sequence of Minimum @@ -221,11 +221,14 @@ TfLiteStatus DecodeAndProcessImage(tflite::ErrorReporter* error_reporter, // See https://en.wikipedia.org/wiki/Grayscale for magic numbers float gray_value = (0.2126 * r) + (0.7152 * g) + (0.0722 * b); + // Convert to signed 8-bit integer by subtracting 128. + gray_value -= 128; + // The x coordinate of this pixel in the output image int current_x = x_origin + mcu_col; // The index of this pixel in our flat output buffer int index = (current_y * image_width) + current_x; - image_data[index] = static_cast(gray_value); + image_data[index] = static_cast(gray_value); } } } @@ -235,7 +238,7 @@ TfLiteStatus DecodeAndProcessImage(tflite::ErrorReporter* error_reporter, // Get an image from the camera module TfLiteStatus GetImage(tflite::ErrorReporter* error_reporter, int image_width, - int image_height, int channels, uint8_t* image_data) { + int image_height, int channels, int8_t* image_data) { static bool g_is_camera_initialized = false; if (!g_is_camera_initialized) { TfLiteStatus init_status = InitCamera(error_reporter); diff --git a/tensorflow/lite/micro/examples/person_detection/detection_responder.cc b/tensorflow/lite/micro/examples/person_detection/detection_responder.cc index 1d7974b22fd..2e3f99bf0e8 100644 --- a/tensorflow/lite/micro/examples/person_detection/detection_responder.cc +++ b/tensorflow/lite/micro/examples/person_detection/detection_responder.cc @@ -19,7 +19,7 @@ limitations under the License. // console. Real applications will want to take some custom action instead, and // should implement their own versions of this function. void RespondToDetection(tflite::ErrorReporter* error_reporter, - uint8_t person_score, uint8_t no_person_score) { + int8_t person_score, int8_t no_person_score) { TF_LITE_REPORT_ERROR(error_reporter, "person score:%d no person score %d", person_score, no_person_score); } diff --git a/tensorflow/lite/micro/examples/person_detection/detection_responder.h b/tensorflow/lite/micro/examples/person_detection/detection_responder.h index a7c709daa38..8887c58f6e9 100644 --- a/tensorflow/lite/micro/examples/person_detection/detection_responder.h +++ b/tensorflow/lite/micro/examples/person_detection/detection_responder.h @@ -29,6 +29,6 @@ limitations under the License. // image is considered to contain a person. This threshold may be adjusted for // particular applications. void RespondToDetection(tflite::ErrorReporter* error_reporter, - uint8_t person_score, uint8_t no_person_score); + int8_t person_score, int8_t no_person_score); #endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_DETECTION_RESPONDER_H_ diff --git a/tensorflow/lite/micro/examples/person_detection/detection_responder_test.cc b/tensorflow/lite/micro/examples/person_detection/detection_responder_test.cc index 7292a4a1ade..6a2198b7679 100644 --- a/tensorflow/lite/micro/examples/person_detection/detection_responder_test.cc +++ b/tensorflow/lite/micro/examples/person_detection/detection_responder_test.cc @@ -25,8 +25,8 @@ TF_LITE_MICRO_TEST(TestCallability) { // This will have external side-effects (like printing to the debug console // or lighting an LED) that are hard to observe, so the most we can do is // make sure the call doesn't crash. - RespondToDetection(µ_error_reporter, 100, 200); - RespondToDetection(µ_error_reporter, 200, 100); + RespondToDetection(µ_error_reporter, -100, 100); + RespondToDetection(µ_error_reporter, 100, 50); } TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/examples/person_detection/himax_driver/HM01B0_optimized.c b/tensorflow/lite/micro/examples/person_detection/himax_driver/HM01B0_optimized.c index 0547ba82cdb..7bc5b2b3be7 100644 --- a/tensorflow/lite/micro/examples/person_detection/himax_driver/HM01B0_optimized.c +++ b/tensorflow/lite/micro/examples/person_detection/himax_driver/HM01B0_optimized.c @@ -45,8 +45,9 @@ static const int kStrideShift = 1; //! @return Error code. // //***************************************************************************** -uint32_t hm01b0_blocking_read_oneframe_scaled( - hm01b0_cfg_t* psCfg, uint8_t* buffer, int w, int h, int channels) { +uint32_t hm01b0_blocking_read_oneframe_scaled(hm01b0_cfg_t* psCfg, + int8_t* buffer, int w, int h, + int channels) { hm01b0_single_frame_capture(psCfg); // Calculate the number of pixels to crop to get a centered image. @@ -76,7 +77,9 @@ uint32_t hm01b0_blocking_read_oneframe_scaled( if (output_x < w && output_y < h) { const int output_idx = (output_y * w + output_x) * channels; for (int i=0; i micro_op_resolver; + static tflite::MicroMutableOpResolver<5> micro_op_resolver; micro_op_resolver.AddAveragePool2D(); micro_op_resolver.AddConv2D(); micro_op_resolver.AddDepthwiseConv2D(); + micro_op_resolver.AddReshape(); + micro_op_resolver.AddSoftmax(); // Build an interpreter to run the model with. + // NOLINTNEXTLINE(runtime-global-variables) static tflite::MicroInterpreter static_interpreter( model, micro_op_resolver, tensor_arena, kTensorArenaSize, error_reporter); interpreter = &static_interpreter; @@ -89,7 +99,7 @@ void setup() { void loop() { // Get image from provider. if (kTfLiteOk != GetImage(error_reporter, kNumCols, kNumRows, kNumChannels, - input->data.uint8)) { + input->data.int8)) { TF_LITE_REPORT_ERROR(error_reporter, "Image capture failed."); } @@ -101,7 +111,7 @@ void loop() { TfLiteTensor* output = interpreter->output(0); // Process the inference results. - uint8_t person_score = output->data.uint8[kPersonIndex]; - uint8_t no_person_score = output->data.uint8[kNotAPersonIndex]; + int8_t person_score = output->data.uint8[kPersonIndex]; + int8_t no_person_score = output->data.uint8[kNotAPersonIndex]; RespondToDetection(error_reporter, person_score, no_person_score); } diff --git a/tensorflow/lite/micro/examples/person_detection/model_settings.cc b/tensorflow/lite/micro/examples/person_detection/model_settings.cc index 99a1899e22b..f11d48aa77c 100644 --- a/tensorflow/lite/micro/examples/person_detection/model_settings.cc +++ b/tensorflow/lite/micro/examples/person_detection/model_settings.cc @@ -16,7 +16,6 @@ limitations under the License. #include "tensorflow/lite/micro/examples/person_detection/model_settings.h" const char* kCategoryLabels[kCategoryCount] = { - "unused", - "person", "notperson", + "person", }; diff --git a/tensorflow/lite/micro/examples/person_detection/model_settings.h b/tensorflow/lite/micro/examples/person_detection/model_settings.h index e666f824c6c..f94d58ed652 100644 --- a/tensorflow/lite/micro/examples/person_detection/model_settings.h +++ b/tensorflow/lite/micro/examples/person_detection/model_settings.h @@ -27,9 +27,9 @@ constexpr int kNumChannels = 1; constexpr int kMaxImageSize = kNumCols * kNumRows * kNumChannels; -constexpr int kCategoryCount = 3; +constexpr int kCategoryCount = 2; constexpr int kPersonIndex = 1; -constexpr int kNotAPersonIndex = 2; +constexpr int kNotAPersonIndex = 0; extern const char* kCategoryLabels[kCategoryCount]; #endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_MODEL_SETTINGS_H_ diff --git a/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc b/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc index ad0c5e6268e..97cef14661a 100644 --- a/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc +++ b/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc @@ -26,8 +26,7 @@ limitations under the License. #include "tensorflow/lite/version.h" // Create an area of memory to use for input, output, and intermediate arrays. -constexpr int tensor_arena_size = 93 * 1024; -__attribute__((section(".bss.NoInit"), aligned(16))) +constexpr int tensor_arena_size = 136 * 1024; uint8_t tensor_arena[tensor_arena_size]; TF_LITE_MICRO_TESTS_BEGIN @@ -51,12 +50,12 @@ TF_LITE_MICRO_TEST(TestInvoke) { // An easier approach is to just use the AllOpsResolver, but this will // incur some penalty in code space for op implementations that are not // needed by this graph. - // - // tflite::AllOpsResolver resolver; - tflite::MicroMutableOpResolver<3> micro_op_resolver; + tflite::MicroMutableOpResolver<5> micro_op_resolver; micro_op_resolver.AddAveragePool2D(); micro_op_resolver.AddConv2D(); micro_op_resolver.AddDepthwiseConv2D(); + micro_op_resolver.AddReshape(); + micro_op_resolver.AddSoftmax(); // Build an interpreter to run the model with. tflite::MicroInterpreter interpreter(model, micro_op_resolver, tensor_arena, @@ -74,13 +73,11 @@ TF_LITE_MICRO_TEST(TestInvoke) { TF_LITE_MICRO_EXPECT_EQ(kNumRows, input->dims->data[1]); TF_LITE_MICRO_EXPECT_EQ(kNumCols, input->dims->data[2]); TF_LITE_MICRO_EXPECT_EQ(kNumChannels, input->dims->data[3]); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteUInt8, input->type); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteInt8, input->type); // Copy an image with a person into the memory area used for the input. - const uint8_t* person_data = g_person_data; - for (size_t i = 0; i < input->bytes; ++i) { - input->data.uint8[i] = person_data[i]; - } + TFLITE_DCHECK_EQ(input->bytes, static_cast(g_person_data_size)); + memcpy(input->data.int8, g_person_data, input->bytes); // Run the model on this input and make sure it succeeds. TfLiteStatus invoke_status = interpreter.Invoke(); @@ -92,26 +89,21 @@ TF_LITE_MICRO_TEST(TestInvoke) { // Get the output from the model, and make sure it's the expected size and // type. TfLiteTensor* output = interpreter.output(0); - TF_LITE_MICRO_EXPECT_EQ(4, output->dims->size); + TF_LITE_MICRO_EXPECT_EQ(2, output->dims->size); TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[0]); - TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[1]); - TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[2]); - TF_LITE_MICRO_EXPECT_EQ(kCategoryCount, output->dims->data[3]); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteUInt8, output->type); + TF_LITE_MICRO_EXPECT_EQ(kCategoryCount, output->dims->data[1]); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteInt8, output->type); // Make sure that the expected "Person" score is higher than the other class. - uint8_t person_score = output->data.uint8[kPersonIndex]; - uint8_t no_person_score = output->data.uint8[kNotAPersonIndex]; + int8_t person_score = output->data.int8[kPersonIndex]; + int8_t no_person_score = output->data.int8[kNotAPersonIndex]; TF_LITE_REPORT_ERROR(µ_error_reporter, "person data. person score: %d, no person score: %d\n", person_score, no_person_score); TF_LITE_MICRO_EXPECT_GT(person_score, no_person_score); - // Now test with a different input, from an image without a person. - const uint8_t* no_person_data = g_no_person_data; - for (size_t i = 0; i < input->bytes; ++i) { - input->data.uint8[i] = no_person_data[i]; - } + // TODO(b/161461076): Update model to make this work on real negative inputs. + memset(input->data.int8, 0, input->bytes); // Run the model on this "No Person" input. invoke_status = interpreter.Invoke(); @@ -123,16 +115,14 @@ TF_LITE_MICRO_TEST(TestInvoke) { // Get the output from the model, and make sure it's the expected size and // type. output = interpreter.output(0); - TF_LITE_MICRO_EXPECT_EQ(4, output->dims->size); + TF_LITE_MICRO_EXPECT_EQ(2, output->dims->size); TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[0]); - TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[1]); - TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[2]); - TF_LITE_MICRO_EXPECT_EQ(kCategoryCount, output->dims->data[3]); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteUInt8, output->type); + TF_LITE_MICRO_EXPECT_EQ(kCategoryCount, output->dims->data[1]); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteInt8, output->type); // Make sure that the expected "No Person" score is higher. - person_score = output->data.uint8[kPersonIndex]; - no_person_score = output->data.uint8[kNotAPersonIndex]; + person_score = output->data.int8[kPersonIndex]; + no_person_score = output->data.int8[kNotAPersonIndex]; TF_LITE_REPORT_ERROR( µ_error_reporter, "no person data. person score: %d, no person score: %d\n", person_score, diff --git a/tensorflow/lite/micro/examples/person_detection/sparkfun_edge/detection_responder.cc b/tensorflow/lite/micro/examples/person_detection/sparkfun_edge/detection_responder.cc index 9025232b215..0ab9a3bdf21 100644 --- a/tensorflow/lite/micro/examples/person_detection/sparkfun_edge/detection_responder.cc +++ b/tensorflow/lite/micro/examples/person_detection/sparkfun_edge/detection_responder.cc @@ -26,7 +26,7 @@ limitations under the License. // This implementation will light up LEDs on the board in response to the // inference results. void RespondToDetection(tflite::ErrorReporter* error_reporter, - uint8_t person_score, uint8_t no_person_score) { + int8_t person_score, int8_t no_person_score) { static bool is_initialized = false; if (!is_initialized) { // Setup LED's as outputs. Leave red LED alone since that's an error diff --git a/tensorflow/lite/micro/examples/person_detection/sparkfun_edge/image_provider.cc b/tensorflow/lite/micro/examples/person_detection/sparkfun_edge/image_provider.cc index 22c52651e7c..fc1c486af43 100644 --- a/tensorflow/lite/micro/examples/person_detection/sparkfun_edge/image_provider.cc +++ b/tensorflow/lite/micro/examples/person_detection/sparkfun_edge/image_provider.cc @@ -190,7 +190,7 @@ TfLiteStatus InitCamera(tflite::ErrorReporter* error_reporter) { // Capture single frame. Frame pointer passed in to reduce memory usage. This // allows the input tensor to be used instead of requiring an extra copy. TfLiteStatus GetImage(tflite::ErrorReporter* error_reporter, int frame_width, - int frame_height, int channels, uint8_t* frame) { + int frame_height, int channels, int8_t* frame) { if (!g_is_camera_initialized) { TfLiteStatus init_status = InitCamera(error_reporter); if (init_status != kTfLiteOk) { diff --git a/tensorflow/lite/micro/examples/person_detection/training_a_model.md b/tensorflow/lite/micro/examples/person_detection/training_a_model.md index f6a20eec9c7..81ac39bbf14 100644 --- a/tensorflow/lite/micro/examples/person_detection/training_a_model.md +++ b/tensorflow/lite/micro/examples/person_detection/training_a_model.md @@ -372,6 +372,9 @@ tf.lite.TFLiteConverter.from_frozen_graph('vww_96_grayscale_frozen.pb', ['input'], ['MobilenetV1/Predictions/Reshape_1']) converter.optimizations = [tf.lite.Optimize.DEFAULT] converter.representative_dataset = representative_dataset_gen +converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] +converter.inference_input_type = tf.int8 +converter.inference_output_type = tf.int8 tflite_quant_model = converter.convert() open("vww_96_grayscale_quantized.tflite", "wb").write(tflite_quant_model) diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/BUILD b/tensorflow/lite/micro/examples/person_detection_experimental/BUILD deleted file mode 100644 index 0a070d5259d..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/BUILD +++ /dev/null @@ -1,138 +0,0 @@ -# Description: -# TensorFlow Lite for Microcontrollers Vision Example. - -load( - "//tensorflow/lite/micro/testing:micro_test.bzl", - "tflite_micro_cc_test", -) - -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) # Apache 2.0 - -cc_library( - name = "model_settings", - srcs = [ - "model_settings.cc", - ], - hdrs = [ - "model_settings.h", - ], -) - -cc_library( - name = "person_detect_model_data", - srcs = [ - "person_detect_model_data.cc", - ], - hdrs = [ - "person_detect_model_data.h", - ], -) - -cc_library( - name = "simple_images_test_data", - srcs = [ - "no_person_image_data.cc", - "person_image_data.cc", - ], - hdrs = [ - "no_person_image_data.h", - "person_image_data.h", - ], - deps = [ - ":model_settings", - ], -) - -tflite_micro_cc_test( - name = "person_detection_test", - srcs = ["person_detection_test.cc"], - deps = [ - ":model_settings", - ":person_detect_model_data", - ":simple_images_test_data", - "//tensorflow/lite:schema_fbs_version", - "//tensorflow/lite/c:common", - "//tensorflow/lite/micro:micro_error_reporter", - "//tensorflow/lite/micro:micro_framework", - "//tensorflow/lite/micro:op_resolvers", - "//tensorflow/lite/micro/testing:micro_test", - "//tensorflow/lite/schema:schema_fbs", - ], -) - -cc_library( - name = "image_provider", - srcs = [ - "image_provider.cc", - ], - hdrs = [ - "image_provider.h", - ], - deps = [ - ":model_settings", - "//tensorflow/lite/c:common", - "//tensorflow/lite/micro:micro_error_reporter", - ], -) - -tflite_micro_cc_test( - name = "image_provider_test", - srcs = [ - "image_provider_test.cc", - ], - deps = [ - ":image_provider", - ":model_settings", - "//tensorflow/lite/c:common", - "//tensorflow/lite/micro:micro_error_reporter", - "//tensorflow/lite/micro:micro_framework", - "//tensorflow/lite/micro/testing:micro_test", - ], -) - -cc_library( - name = "detection_responder", - srcs = [ - "detection_responder.cc", - ], - hdrs = [ - "detection_responder.h", - ], - deps = [ - "//tensorflow/lite/c:common", - "//tensorflow/lite/micro:micro_error_reporter", - ], -) - -tflite_micro_cc_test( - name = "detection_responder_test", - srcs = [ - "detection_responder_test.cc", - ], - deps = [ - ":detection_responder", - "//tensorflow/lite/micro/testing:micro_test", - ], -) - -cc_binary( - name = "person_detection", - srcs = [ - "main.cc", - "main_functions.cc", - "main_functions.h", - ], - deps = [ - ":detection_responder", - ":image_provider", - ":model_settings", - ":person_detect_model_data", - "//tensorflow/lite:schema_fbs_version", - "//tensorflow/lite/micro:micro_error_reporter", - "//tensorflow/lite/micro:micro_framework", - "//tensorflow/lite/micro:op_resolvers", - "//tensorflow/lite/schema:schema_fbs", - ], -) diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/Makefile.inc b/tensorflow/lite/micro/examples/person_detection_experimental/Makefile.inc deleted file mode 100644 index 3e242878a79..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/Makefile.inc +++ /dev/null @@ -1,83 +0,0 @@ -$(eval $(call add_third_party_download,$(PERSON_MODEL_INT8_URL),$(PERSON_MODEL_INT8_MD5),person_model_int8,)) - -person_detection_MODEL_SRCS := \ -tensorflow/lite/micro/examples/person_detection_experimental/model_settings.cc \ -$(MAKEFILE_DIR)/downloads/person_model_int8/person_detect_model_data.cc - -person_detection_MODEL_HDRS := \ -tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h \ -tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h - -person_detection_TEST_SRCS := \ -tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc \ -$(MAKEFILE_DIR)/downloads/person_model_int8/no_person_image_data.cc \ -$(MAKEFILE_DIR)/downloads/person_model_int8/person_image_data.cc \ -$(person_detection_MODEL_SRCS) - -person_detection_TEST_HDRS := \ -tensorflow/lite/micro/examples/person_detection_experimental/no_person_image_data.h \ -tensorflow/lite/micro/examples/person_detection_experimental/person_image_data.h \ -$(person_detection_MODEL_HDRS) - -IMAGE_PROVIDER_TEST_SRCS := \ -tensorflow/lite/micro/examples/person_detection_experimental/image_provider.cc \ -tensorflow/lite/micro/examples/person_detection_experimental/image_provider_test.cc \ -tensorflow/lite/micro/examples/person_detection_experimental/model_settings.cc - -IMAGE_PROVIDER_TEST_HDRS := \ -tensorflow/lite/micro/examples/person_detection_experimental/image_provider.h \ -tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h - -DETECTION_RESPONDER_TEST_SRCS := \ -tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.cc \ -tensorflow/lite/micro/examples/person_detection_experimental/detection_responder_test.cc - -DETECTION_RESPONDER_TEST_HDRS := \ -tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h - -person_detection_SRCS := \ -tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.cc \ -tensorflow/lite/micro/examples/person_detection_experimental/image_provider.cc \ -tensorflow/lite/micro/examples/person_detection_experimental/main.cc \ -tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc \ -$(person_detection_MODEL_SRCS) - -person_detection_HDRS := \ -tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h \ -tensorflow/lite/micro/examples/person_detection_experimental/image_provider.h \ -tensorflow/lite/micro/examples/person_detection_experimental/main_functions.h \ -$(person_detection_MODEL_HDRS) - -#Find any platform - specific rules for this example. -include $(wildcard tensorflow/lite/micro/examples/person_detection_experimental/*/Makefile.inc) - -# Tests loading and running a vision model. -$(eval $(call microlite_test,person_detection_test_int8,\ -$(person_detection_TEST_SRCS),$(person_detection_TEST_HDRS))) - -# Three conflicting issues here: -# 1. The image_provider_test fails on Sparkfun Edge we do not have a way to -# filter out individual tests within and example. -# 2. We do not want to completely remove person_detection from the sparkfun_edge -# build. -# 3. We do want to keep as many targets as possible be part of the sparkfun_edge -# CI build to avoid getting into similar situations where some parts of the -# code are supported on a platform while other parts are not. -# -# The current nasty workaround is to explicitly exclude the offending test for -# the sparkfun_edge target. Note that we are not exluding it for -# TARGET=apollo3evb becuase that is not part of our CI builds (and the two are -# basically equivalent). -ifneq ($(TARGET),sparkfun_edge) -# Tests the image provider module. -$(eval $(call microlite_test,image_provider_test_int8,\ -$(IMAGE_PROVIDER_TEST_SRCS),$(IMAGE_PROVIDER_TEST_HDRS))) -endif - -# Tests the detection responder module. -$(eval $(call microlite_test,detection_responder_test_int8,\ -$(DETECTION_RESPONDER_TEST_SRCS),$(DETECTION_RESPONDER_TEST_HDRS))) - -# Builds a standalone object recognition binary. -$(eval $(call microlite_test,person_detection_int8,\ -$(person_detection_SRCS),$(person_detection_HDRS))) diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/README.md b/tensorflow/lite/micro/examples/person_detection_experimental/README.md deleted file mode 100644 index c628cfbc9ce..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/README.md +++ /dev/null @@ -1,570 +0,0 @@ -# Person detection example - -This example shows how you can use Tensorflow Lite to run a 250 kilobyte neural -network to recognize people in images captured by a camera. It is designed to -run on systems with small amounts of memory such as microcontrollers and DSPs. -This uses the experimental int8 quantized version of the person detection model. - -## Table of contents - -- [Getting started](#getting-started) -- [Running on ARC EM SDP](#running-on-arc-em-sdp) -- [Running on Arduino](#running-on-arduino) -- [Running on HIMAX WE1 EVB](#running-on-himax-we1-evb) -- [Running on SparkFun Edge](#running-on-sparkfun-edge) -- [Run the tests on a development machine](#run-the-tests-on-a-development-machine) -- [Debugging image capture](#debugging-image-capture) -- [Training your own model](#training-your-own-model) - -## Running on ARC EM SDP - -The following instructions will help you to build and deploy this example to -[ARC EM SDP](https://www.synopsys.com/dw/ipdir.php?ds=arc-em-software-development-platform) -board. General information and instructions on using the board with TensorFlow -Lite Micro can be found in the common -[ARC targets description](/tensorflow/lite/micro/tools/make/targets/arc/README.md). - -This example uses asymmetric int8 quantization and can therefore leverage -optimized int8 kernels from the embARC MLI library - -The ARC EM SDP board contains a rich set of extension interfaces. You can choose -any compatible camera and modify -[image_provider.cc](/tensorflow/lite/micro/examples/person_detection_experimental/image_provider.cc) -file accordingly to use input from your specific camera. By default, results of -running this example are printed to the console. If you would like to instead -implement some target-specific actions, you need to modify -[detection_responder.cc](/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.cc) -accordingly. - -The reference implementations of these files are used by default on the EM SDP. - -### Initial setup - -Follow the instructions on the -[ARC EM SDP Initial Setup](/tensorflow/lite/micro/tools/make/targets/arc/README.md#ARC-EM-Software-Development-Platform-ARC-EM-SDP) -to get and install all required tools for work with ARC EM SDP. - -### Generate Example Project - -The example project for ARC EM SDP platform can be generated with the following -command: - -``` -make -f tensorflow/lite/micro/tools/make/Makefile \ -TARGET=arc_emsdp TAGS=reduce_codesize \ -generate_person_detection_int8_make_project -``` - -Note that `TAGS=reduce_codesize` applies example specific changes of code to -reduce total size of application. It can be omitted. - -### Build and Run Example - -For more detailed information on building and running examples see the -appropriate sections of general descriptions of the -[ARC EM SDP usage with TFLM](/tensorflow/lite/micro/tools/make/targets/arc/README.md#ARC-EM-Software-Development-Platform-ARC-EM-SDP). -In the directory with generated project you can also find a -*README_ARC_EMSDP.md* file with instructions and options on building and -running. Here we only briefly mention main steps which are typically enough to -get it started. - -1. You need to - [connect the board](/tensorflow/lite/micro/tools/make/targets/arc/README.md#connect-the-board) - and open an serial connection. - -2. Go to the generated example project director - - ``` - cd tensorflow/lite/micro/tools/make/gen/arc_emsdp_arc/prj/person_detection_int8/make - ``` - -3. Build the example using - - ``` - make app - ``` - -4. To generate artefacts for self-boot of example from the board use - - ``` - make flash - ``` - -5. To run application from the board using microSD card: - - * Copy the content of the created /bin folder into the root of microSD - card. Note that the card must be formatted as FAT32 with default cluster - size (but less than 32 Kbytes) - * Plug in the microSD card into the J11 connector. - * Push the RST button. If a red LED is lit beside RST button, push the CFG - button. - * Type or copy next commands one-by-another into serial terminal: `setenv - loadaddr 0x10800000 setenv bootfile app.elf setenv bootdelay 1 setenv - bootcmd fatload mmc 0 \$\{loadaddr\} \$\{bootfile\} \&\& bootelf - saveenv` - * Push the RST button. - -6. If you have the MetaWare Debugger installed in your environment: - - * To run application from the console using it type `make run`. - * To stop the execution type `Ctrl+C` in the console several times. - -In both cases (step 5 and 6) you will see the application output in the serial -terminal. - -## Running on Arduino - -The following instructions will help you build and deploy this sample -to [Arduino](https://www.arduino.cc/) devices. - -The sample has been tested with the following device: - -- [Arduino Nano 33 BLE Sense](https://store.arduino.cc/usa/nano-33-ble-sense-with-headers) - -You will also need the following camera module: - -- [Arducam Mini 2MP Plus](https://www.amazon.com/Arducam-Module-Megapixels-Arduino-Mega2560/dp/B012UXNDOY) - -### Hardware - -Connect the Arducam pins as follows: - -|Arducam pin name|Arduino pin name| -|----------------|----------------| -|CS|D7 (unlabelled, immediately to the right of D6)| -|MOSI|D11| -|MISO|D12| -|SCK|D13| -|GND|GND (either pin marked GND is fine)| -|VCC|3.3 V| -|SDA|A4| -|SCL|A5| - -### Install the Arduino_TensorFlowLite library - -Download the current nightly build of the library: -[person_detection.zip](https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_int8_grayscale_2020_01_13.zip) - -This example application is included as part of the official TensorFlow Lite -Arduino library. To install it, open the Arduino library manager in -`Tools -> Manage Libraries...` and search for `Arduino_TensorFlowLite`. - -### Install other libraries - -In addition to the TensorFlow library, you'll also need to install two -libraries: - -* The Arducam library, so our code can interface with the hardware -* The JPEGDecoder library, so we can decode JPEG-encoded images - -The Arducam Arduino library is available from GitHub at -[https://github.com/ArduCAM/Arduino](https://github.com/ArduCAM/Arduino). -To install it, download or clone the repository. Next, copy its `ArduCAM` -subdirectory into your `Arduino/libraries` directory. To find this directory on -your machine, check the *Sketchbook location* in the Arduino IDE's -*Preferences* window. - -After downloading the library, you'll need to edit one of its files to make sure -it is configured for the Arducam Mini 2MP Plus. To do so, open the following -file: - -``` -Arduino/libraries/ArduCAM/memorysaver.h -``` - -You'll see a bunch of `#define` statements listed. Make sure that they are all -commented out, except for `#define OV2640_MINI_2MP_PLUS`, as so: - -``` -//Step 1: select the hardware platform, only one at a time -//#define OV2640_MINI_2MP -//#define OV3640_MINI_3MP -//#define OV5642_MINI_5MP -//#define OV5642_MINI_5MP_BIT_ROTATION_FIXED -#define OV2640_MINI_2MP_PLUS -//#define OV5642_MINI_5MP_PLUS -//#define OV5640_MINI_5MP_PLUS -``` - -Once you save the file, we're done configuring the Arducam library. - -Our next step is to install the JPEGDecoder library. We can do this from within -the Arduino IDE. First, go to the *Manage Libraries...* option in the *Tools* -menu and search for `JPEGDecoder`. You should install version _1.8.0_ of the -library. - -Once the library has installed, we'll need to configure it to disable some -optional components that are not compatible with the Arduino Nano 33 BLE Sense. -Open the following file: - -``` -Arduino/libraries/JPEGDecoder/src/User_Config.h -``` - -Make sure that both `#define LOAD_SD_LIBRARY` and `#define LOAD_SDFAT_LIBRARY` -are commented out, as shown in this excerpt from the file: - -```c++ -// Comment out the next #defines if you are not using an SD Card to store the JPEGs -// Commenting out the line is NOT essential but will save some FLASH space if -// SD Card access is not needed. Note: use of SdFat is currently untested! - -//#define LOAD_SD_LIBRARY // Default SD Card library -//#define LOAD_SDFAT_LIBRARY // Use SdFat library instead, so SD Card SPI can be bit bashed -``` - -Once you've saved the file, you are done installing libraries. - -### Load and run the example - -Go to `File -> Examples`. You should see an -example near the bottom of the list named `TensorFlowLite`. Select -it and click `person_detection` to load the example. Connect your device, then -build and upload the example. - -To test the camera, start by pointing the device's camera at something that is -definitely not a person, or just covering it up. The next time the blue LED -flashes, the device will capture a frame from the camera and begin to run -inference. Since the vision model we are using for person detection is -relatively large, it takes a long time to run inference—around 19 seconds at the -time of writing, though it's possible TensorFlow Lite has gotten faster since -then. - -After 19 seconds or so, the inference result will be translated into another LED -being lit. Since you pointed the camera at something that isn't a person, the -red LED should light up. - -Now, try pointing the device's camera at yourself! The next time the blue LED -flashes, the device will capture another image and begin to run inference. After -19 seconds, the green LED should light up! - -Remember, image data is captured as a snapshot before each inference, whenever -the blue LED flashes. Whatever the camera is pointed at during that moment is -what will be fed into the model. It doesn't matter where the camera is pointed -until the next time an image is captured, when the blue LED will flash again. - -If you're getting seemingly incorrect results, make sure you are in an -environment with good lighting. You should also make sure that the camera is -oriented correctly, with the pins pointing downwards, so that the images it -captures are the right way up—the model was not trained to recognize upside-down -people! In addition, it's good to remember that this is a tiny model, which -trades accuracy for small size. It works very well, but it isn't accurate 100% -of the time. - -We can also see the results of inference via the Arduino Serial Monitor. To do -this, open the *Serial Monitor* from the *Tools* menu. You'll see a detailed -log of what is happening while our application runs. It's also interesting to -check the *Show timestamp* box, so you can see how long each part of the process -takes: - -``` -14:17:50.714 -> Starting capture -14:17:50.714 -> Image captured -14:17:50.784 -> Reading 3080 bytes from ArduCAM -14:17:50.887 -> Finished reading -14:17:50.887 -> Decoding JPEG and converting to greyscale -14:17:51.074 -> Image decoded and processed -14:18:09.710 -> Person score: 246 No person score: 66 -``` - -From the log, we can see that it took around 170 ms to capture and read the -image data from the camera module, 180 ms to decode the JPEG and convert it to -greyscale, and 18.6 seconds to run inference. - -## Running on HIMAX WE1 EVB - -The following instructions will help you build and deploy this example to -[HIMAX WE1 EVB](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_board_brief) -board. To understand more about using this board, please check -[HIMAX WE1 EVB user guide](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide). - -### Initial Setup - -To use the HIMAX WE1 EVB, please make sure following software are installed: - -#### MetaWare Development Toolkit - -See -[Install the Synopsys DesignWare ARC MetaWare Development Toolkit](/tensorflow/lite/micro/tools/make/targets/arc/README.md#install-the-synopsys-designware-arc-metaware-development-toolkit) -section for instructions on toolchain installation. - -#### Make Tool version - -A `'make'` tool is required for deploying Tensorflow Lite Micro applications on -HIMAX WE1 EVB, See -[Check make tool version](/tensorflow/lite/micro/tools/make/targets/arc/README.md#make-tool) -section for proper environment. - -#### Serial Terminal Emulation Application - -There are 2 main purposes for HIMAX WE1 EVB Debug UART port - -- print application output -- burn application to flash by using xmodem send application binary - -You can use any terminal emulation program (like [PuTTY](https://www.putty.org/) -or [minicom](https://linux.die.net/man/1/minicom)). - -### Generate Example Project - -The example project for HIMAX WE1 EVB platform can be generated with the -following command: - -Download related third party data - -``` -make -f tensorflow/lite/micro/tools/make/Makefile TARGET=himax_we1_evb third_party_downloads -``` - -Generate person detection project - -``` -make -f tensorflow/lite/micro/tools/make/Makefile generate_person_detection_int8_make_project TARGET=himax_we1_evb -``` - -### Build and Burn Example - -Following the Steps to run person detection example at HIMAX WE1 EVB platform. - -1. Go to the generated example project directory. - - ``` - cd tensorflow/lite/micro/tools/make/gen/himax_we1_evb_arc/prj/person_detection_int8/make - ``` - -2. Build the example using - - ``` - make app - ``` - -3. After example build finish, copy ELF file and map file to image generate - tool directory. \ - image generate tool directory located at - `'tensorflow/lite/micro/tools/make/downloads/himax_we1_sdk/image_gen_linux_v3/'` - - ``` - cp person_detection_int8.elf himax_we1_evb.map ../../../../../downloads/himax_we1_sdk/image_gen_linux_v3/ - ``` - -4. Go to flash image generate tool directory. - - ``` - cd ../../../../../downloads/himax_we1_sdk/image_gen_linux_v3/ - ``` - - make sure this tool directory is in $PATH. You can permanently set it to - PATH by - - ``` - export PATH=$PATH:$(pwd) - ``` - -5. run image generate tool, generate flash image file. - - * Before running image generate tool, by typing `sudo chmod +x image_gen` - and `sudo chmod +x sign_tool` to make sure it is executable. - - ``` - image_gen -e person_detection_int8.elf -m himax_we1_evb.map -o out.img - ``` - -6. Download flash image file to HIMAX WE1 EVB by UART: - - * more detail about download image through UART can be found at - [HIMAX WE1 EVB update Flash image](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide#flash-image-update) - -After these steps, press reset button on the HIMAX WE1 EVB, you will see -application output in the serial terminal. - -## Running on SparkFun Edge - -The following instructions will help you build and deploy this sample on the -[SparkFun Edge development board](https://sparkfun.com/products/15170). This -sample requires the Sparkfun Himax camera for the Sparkfun Edge board. It is -not available for purchase yet. - -If you're new to using this board, we recommend walking through the -[AI on a microcontroller with TensorFlow Lite and SparkFun Edge](https://codelabs.developers.google.com/codelabs/sparkfun-tensorflow) -codelab to get an understanding of the workflow. - -### Compile the binary - -The following command will download the required dependencies and then compile a -binary for the SparkFun Edge: - -``` -make -f tensorflow/lite/micro/tools/make/Makefile TARGET=sparkfun_edge person_detection_bin -``` - -The binary will be created in the following location: - -``` -tensorflow/lite/micro/tools/make/gen/sparkfun_edge_cortex-m4/bin/person_detection.bin -``` - -### Sign the binary - -The binary must be signed with cryptographic keys to be deployed to the device. -We'll now run some commands that will sign our binary so it can be flashed to -the SparkFun Edge. The scripts we are using come from the Ambiq SDK, which is -downloaded when the `Makefile` is run. - -Enter the following command to set up some dummy cryptographic keys we can use -for development: - -``` -cp tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/keys_info0.py \ -tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/keys_info.py -``` - -Next, run the following command to create a signed binary: - -``` -python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/create_cust_image_blob.py \ ---bin tensorflow/lite/micro/tools/make/gen/sparkfun_edge_cortex-m4/bin/person_detection.bin \ ---load-address 0xC000 \ ---magic-num 0xCB \ --o main_nonsecure_ota \ ---version 0x0 -``` - -This will create the file `main_nonsecure_ota.bin`. We'll now run another -command to create a final version of the file that can be used to flash our -device with the bootloader script we will use in the next step: - -``` -python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/create_cust_wireupdate_blob.py \ ---load-address 0x20000 \ ---bin main_nonsecure_ota.bin \ --i 6 \ --o main_nonsecure_wire \ ---options 0x1 -``` - -You should now have a file called `main_nonsecure_wire.bin` in the directory -where you ran the commands. This is the file we'll be flashing to the device. - -### Flash the binary - -Next, attach the board to your computer via a USB-to-serial adapter. - -**Note:** If you're using the [SparkFun Serial Basic Breakout](https://www.sparkfun.com/products/15096), -you should [install the latest drivers](https://learn.sparkfun.com/tutorials/sparkfun-serial-basic-ch340c-hookup-guide#drivers-if-you-need-them) -before you continue. - -Once connected, assign the USB device name to an environment variable: - -``` -export DEVICENAME=put your device name here -``` - -Set another variable with the baud rate: - -``` -export BAUD_RATE=921600 -``` - -Now, hold the button marked `14` on the device. While still holding the button, -hit the button marked `RST`. Continue holding the button marked `14` while -running the following command: - -``` -python3 tensorflow/lite/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/uart_wired_update.py \ --b ${BAUD_RATE} ${DEVICENAME} \ --r 1 \ --f main_nonsecure_wire.bin \ --i 6 -``` - -You should see a long stream of output as the binary is flashed to the device. -Once you see the following lines, flashing is complete: - -``` -Sending Reset Command. -Done. -``` - -If you don't see these lines, flashing may have failed. Try running through the -steps in [Flash the binary](#flash-the-binary) again (you can skip over setting -the environment variables). If you continue to run into problems, follow the -[AI on a microcontroller with TensorFlow Lite and SparkFun Edge](https://codelabs.developers.google.com/codelabs/sparkfun-tensorflow) -codelab, which includes more comprehensive instructions for the flashing -process. - -The binary should now be deployed to the device. Hit the button marked `RST` to -reboot the board. You should see the device's four LEDs flashing in sequence. - -Debug information is logged by the board while the program is running. To view -it, establish a serial connection to the board using a baud rate of `115200`. -On OSX and Linux, the following command should work: - -``` -screen ${DEVICENAME} 115200 -``` - -To stop viewing the debug output with `screen`, hit `Ctrl+A`, immediately -followed by the `K` key, then hit the `Y` key. - -## Run the tests on a development machine - -To compile and test this example on a desktop Linux or MacOS machine, download -[the TensorFlow source code](https://github.com/tensorflow/tensorflow), `cd` -into the source directory from a terminal, and then run the following command: - -``` -make -f tensorflow/lite/micro/tools/make/Makefile -``` - -This will take a few minutes, and downloads frameworks the code uses like -[CMSIS](https://developer.arm.com/embedded/cmsis) and -[flatbuffers](https://google.github.io/flatbuffers/). Once that process has -finished, run: - -``` -make -f tensorflow/lite/micro/tools/make/Makefile test_person_detection_test -``` - -You should see a series of files get compiled, followed by some logging output -from a test, which should conclude with `~~~ALL TESTS PASSED~~~`. If you see -this, it means that a small program has been built and run that loads a trained -TensorFlow model, runs some example images through it, and got the expected -outputs. This particular test runs images with a and without a person in them, -and checks that the network correctly identifies them. - -To understand how TensorFlow Lite does this, you can look at the `TestInvoke()` -function in -[person_detection_test.cc](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/micro/examples/person_detection/person_detection_test.cc). -It's a fairly small amount of code, creating an interpreter, getting a handle to -a model that's been compiled into the program, and then invoking the interpreter -with the model and sample inputs. - -## Debugging image capture -When the sample is running, check the LEDs to determine whether the inference is -running correctly. If the red light is stuck on, it means there was an error -communicating with the camera. This is likely due to an incorrectly connected -or broken camera. - -During inference, the blue LED will toggle every time inference is complete. The -orange LED indicates that no person was found, and the green LED indicates a -person was found. The red LED should never turn on, since it indicates an error. - -In order to view the captured image, set the DUMP_IMAGE define in main.cc.  This -causes the board to log raw image info to the console. After the board has been -flashed and reset, dump the log to a text file: - - -``` -screen -L -Logfile ${DEVICENAME} 115200 -``` - -Next, run the raw to bitmap converter to view captured images: - -``` -python3 raw_to_bitmap.py -r GRAY -i -``` - -## Training your own model - -You can train your own model with some easy-to-use scripts. See -[training_a_model.md](training_a_model.md) for instructions. diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/apollo3evb/image_provider.cc b/tensorflow/lite/micro/examples/person_detection_experimental/apollo3evb/image_provider.cc deleted file mode 100644 index 3947661e214..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/apollo3evb/image_provider.cc +++ /dev/null @@ -1,198 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/image_provider.h" - -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_RAW8_QVGA_8bits_lsb_5fps.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_optimized.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/platform_Sparkfun_Edge.h" - -// These are headers from Ambiq's Apollo3 SDK. -#include "am_bsp.h" // NOLINT -#include "am_mcu_apollo.h" // NOLINT -#include "am_util.h" // NOLINT - -// #define DEMO_HM01B0_FRAMEBUFFER_DUMP_ENABLE - -// Enabling logging increases power consumption by preventing low power mode -// from being enabled. -#define ENABLE_LOGGING - -namespace { - -//***************************************************************************** -// -// HM01B0 Configuration -// -//***************************************************************************** -static hm01b0_cfg_t s_HM01B0Cfg = { - // i2c settings - ui16SlvAddr : HM01B0_DEFAULT_ADDRESS, - eIOMMode : HM01B0_IOM_MODE, - ui32IOMModule : HM01B0_IOM_MODULE, - sIOMCfg : { - eInterfaceMode : HM01B0_IOM_MODE, - ui32ClockFreq : HM01B0_I2C_CLOCK_FREQ, - }, - pIOMHandle : NULL, - - // MCLK settings - ui32CTimerModule : HM01B0_MCLK_GENERATOR_MOD, - ui32CTimerSegment : HM01B0_MCLK_GENERATOR_SEG, - ui32CTimerOutputPin : HM01B0_PIN_MCLK, - - // data interface - ui8PinSCL : HM01B0_PIN_SCL, - ui8PinSDA : HM01B0_PIN_SDA, - ui8PinD0 : HM01B0_PIN_D0, - ui8PinD1 : HM01B0_PIN_D1, - ui8PinD2 : HM01B0_PIN_D2, - ui8PinD3 : HM01B0_PIN_D3, - ui8PinD4 : HM01B0_PIN_D4, - ui8PinD5 : HM01B0_PIN_D5, - ui8PinD6 : HM01B0_PIN_D6, - ui8PinD7 : HM01B0_PIN_D7, - ui8PinVSYNC : HM01B0_PIN_VSYNC, - ui8PinHSYNC : HM01B0_PIN_HSYNC, - ui8PinPCLK : HM01B0_PIN_PCLK, - - ui8PinTrig : HM01B0_PIN_TRIG, - ui8PinInt : HM01B0_PIN_INT, - pfnGpioIsr : NULL, -}; - -static constexpr int kFramesToInitialize = 4; - -bool g_is_camera_initialized = false; - -void boost_mode_enable(tflite::ErrorReporter* error_reporter, bool bEnable) { - am_hal_burst_avail_e eBurstModeAvailable; - am_hal_burst_mode_e eBurstMode; - - // Check that the Burst Feature is available. - if (AM_HAL_STATUS_SUCCESS == - am_hal_burst_mode_initialize(&eBurstModeAvailable)) { - if (AM_HAL_BURST_AVAIL == eBurstModeAvailable) { - TF_LITE_REPORT_ERROR(error_reporter, "Apollo3 Burst Mode is Available\n"); - } else { - TF_LITE_REPORT_ERROR(error_reporter, - "Apollo3 Burst Mode is Not Available\n"); - return; - } - } else { - TF_LITE_REPORT_ERROR(error_reporter, - "Failed to Initialize for Burst Mode operation\n"); - } - - // Make sure we are in "Normal" mode. - if (AM_HAL_STATUS_SUCCESS == am_hal_burst_mode_disable(&eBurstMode)) { - if (AM_HAL_NORMAL_MODE == eBurstMode) { - TF_LITE_REPORT_ERROR(error_reporter, - "Apollo3 operating in Normal Mode (48MHz)\n"); - } - } else { - TF_LITE_REPORT_ERROR(error_reporter, - "Failed to Disable Burst Mode operation\n"); - } - - // Put the MCU into "Burst" mode. - if (bEnable) { - if (AM_HAL_STATUS_SUCCESS == am_hal_burst_mode_enable(&eBurstMode)) { - if (AM_HAL_BURST_MODE == eBurstMode) { - TF_LITE_REPORT_ERROR(error_reporter, - "Apollo3 operating in Burst Mode (96MHz)\n"); - } - } else { - TF_LITE_REPORT_ERROR(error_reporter, - "Failed to Enable Burst Mode operation\n"); - } - } -} - -} // namespace - -TfLiteStatus InitCamera(tflite::ErrorReporter* error_reporter) { - TF_LITE_REPORT_ERROR(error_reporter, "Initializing HM01B0...\n"); - - am_hal_clkgen_control(AM_HAL_CLKGEN_CONTROL_SYSCLK_MAX, 0); - - // Set the default cache configuration - am_hal_cachectrl_config(&am_hal_cachectrl_defaults); - am_hal_cachectrl_enable(); - - // Configure the board for low power operation. This breaks logging by - // turning off the itm and uart interfaces. -#ifndef ENABLE_LOGGING - am_bsp_low_power_init(); -#endif - - // Enable interrupts so we can receive messages from the boot host. - am_hal_interrupt_master_enable(); - - boost_mode_enable(error_reporter, true); - - hm01b0_power_up(&s_HM01B0Cfg); - - am_util_delay_ms(1); - - hm01b0_mclk_enable(&s_HM01B0Cfg); - - am_util_delay_ms(1); - - hm01b0_init_if(&s_HM01B0Cfg); - - hm01b0_init_system(&s_HM01B0Cfg, (hm_script_t*)sHM01B0InitScript, - sizeof(sHM01B0InitScript) / sizeof(hm_script_t)); - - // Put camera into streaming mode - this makes it so that the camera - // constantly captures images. It is still OK to read and image since the - // camera uses a double-buffered input. This means there is always one valid - // image to read while the other buffer fills. Streaming mode allows the - // camera to perform auto exposure constantly. - hm01b0_set_mode(&s_HM01B0Cfg, HM01B0_REG_MODE_SELECT_STREAMING, 0); - - return kTfLiteOk; -} - -// Capture single frame. Frame pointer passed in to reduce memory usage. This -// allows the input tensor to be used instead of requiring an extra copy. -TfLiteStatus GetImage(tflite::ErrorReporter* error_reporter, int frame_width, - int frame_height, int channels, uint8_t* frame) { - if (!g_is_camera_initialized) { - TfLiteStatus init_status = InitCamera(error_reporter); - if (init_status != kTfLiteOk) { - return init_status; - } - // Drop a few frames until auto exposure is calibrated. - for (int i = 0; i < kFramesToInitialize; ++i) { - hm01b0_blocking_read_oneframe_scaled(frame, frame_width, frame_height, - channels); - } - g_is_camera_initialized = true; - } - - hm01b0_blocking_read_oneframe_scaled(frame, frame_width, frame_height, - channels); - -#ifdef DEMO_HM01B0_FRAMEBUFFER_DUMP_ENABLE - // Allow some time to see result of previous inference before dumping image. - am_util_delay_ms(2000); - hm01b0_framebuffer_dump(frame, frame_width * frame_height * channels); -#endif - - return kTfLiteOk; -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/arc_emsdp/Makefile.inc b/tensorflow/lite/micro/examples/person_detection_experimental/arc_emsdp/Makefile.inc deleted file mode 100644 index 0ecfdfa8737..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/arc_emsdp/Makefile.inc +++ /dev/null @@ -1,43 +0,0 @@ -ifeq ($(TARGET), arc_emsdp) - -# Patch of arc make project to adjust it specifically -# for experimental person detection example. In particular: -# - Use Linker command file with better usage of fast memory -# - Stripout TFLM reference code by default. -# - Optional: replace mli switchers with specialized kernels -# for smaller code size - - person_detection_HDRS += \ - person_detection_int8_patch.txt - - person_detection_TEST_HDRS += \ - person_detection_int8_patch.txt - - ARC_MLI_BACKEND_PATH = /tensorflow/lite/micro/kernels/arc_mli - -# Apply changes in generated project files. -# See related comment echoed (@echo ) after each change -# to get understanding on it's purpose. -%/person_detection_int8_patch.txt: %/emsdp.lcf %/Makefile %$(ARC_MLI_BACKEND_PATH)/conv.cc %$(ARC_MLI_BACKEND_PATH)/depthwise_conv.cc %$(ARC_MLI_BACKEND_PATH)/pooling.cc - @cp tensorflow/lite/micro/examples/person_detection_experimental/arc_emsdp/emsdp.lcf $< - @echo emsdp.lcf: Replace with example specific memory map > $@ - - @sed -E -i 's#MLI_ONLY *\?= *false#MLI_ONLY \?= true#' $(word 2, $^) - @echo Makefile: No Reference fallback for MLI supported functions >> $@ - -ifneq ($(filter $(ALL_TAGS), reduce_codesize),) -# In case 'reduce_codesize' tag is present, we replace common MLI functions with -# specializations appropriate for this particular graph. But such changes of code -# with high probability may not be acceptable for other graphs and will need -# to be adjusted by the user - - @sed -E -i 's#mli_krn_conv2d_nhwc_sa8_sa8_sa32#mli_krn_conv2d_nhwc_sa8_sa8_sa32_k1x1_nopad#' $(word 3, $^) - @sed -E -i 's#mli_krn_depthwise_conv2d_hwcn_sa8_sa8_sa32#mli_krn_depthwise_conv2d_hwcn_sa8_sa8_sa32_k3x3_krnpad#' $(word 4, $^) - @sed -E -i 's#mli_krn_avepool_hwc_sa8#mli_krn_avepool_hwc_sa8_k3x3_nopad#' $(word 5, $^) - @sed -E -i 's#mli_krn_maxpool_hwc_sa8\(in_ptr, \&cfg, out_ptr\);#return kTfLiteError;#' $(word 5, $^) - @echo $(word 3, $^): Use specialization >> $@ - @echo $(word 4, $^): Use specialization >> $@ - @echo $(word 5, $^): Use specialization and remove max pooling >> $@ -endif - -endif diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/arduino/detection_responder.cc b/tensorflow/lite/micro/examples/person_detection_experimental/arduino/detection_responder.cc deleted file mode 100644 index b2d92d816ec..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/arduino/detection_responder.cc +++ /dev/null @@ -1,56 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h" - -#include "Arduino.h" - -// Flash the blue LED after each inference -void RespondToDetection(tflite::ErrorReporter* error_reporter, - int8_t person_score, int8_t no_person_score) { - static bool is_initialized = false; - if (!is_initialized) { - // Pins for the built-in RGB LEDs on the Arduino Nano 33 BLE Sense - pinMode(LEDR, OUTPUT); - pinMode(LEDG, OUTPUT); - pinMode(LEDB, OUTPUT); - is_initialized = true; - } - - // Note: The RGB LEDs on the Arduino Nano 33 BLE - // Sense are on when the pin is LOW, off when HIGH. - - // Switch the person/not person LEDs off - digitalWrite(LEDG, HIGH); - digitalWrite(LEDR, HIGH); - - // Flash the blue LED after every inference. - digitalWrite(LEDB, LOW); - delay(100); - digitalWrite(LEDB, HIGH); - - // Switch on the green LED when a person is detected, - // the red when no person is detected - if (person_score > no_person_score) { - digitalWrite(LEDG, LOW); - digitalWrite(LEDR, HIGH); - } else { - digitalWrite(LEDG, HIGH); - digitalWrite(LEDR, LOW); - } - - TF_LITE_REPORT_ERROR(error_reporter, "Person score: %d No person score: %d", - person_score, no_person_score); -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/arduino/image_provider.cc b/tensorflow/lite/micro/examples/person_detection_experimental/arduino/image_provider.cc deleted file mode 100644 index a65b7f3f1f6..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/arduino/image_provider.cc +++ /dev/null @@ -1,266 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/image_provider.h" - -/* - * The sample requires the following third-party libraries to be installed and - * configured: - * - * Arducam - * ------- - * 1. Download https://github.com/ArduCAM/Arduino and copy its `ArduCAM` - * subdirectory into `Arduino/libraries`. Commit #e216049 has been tested - * with this code. - * 2. Edit `Arduino/libraries/ArduCAM/memorysaver.h` and ensure that - * "#define OV2640_MINI_2MP_PLUS" is not commented out. Ensure all other - * defines in the same section are commented out. - * - * JPEGDecoder - * ----------- - * 1. Install "JPEGDecoder" 1.8.0 from the Arduino library manager. - * 2. Edit "Arduino/Libraries/JPEGDecoder/src/User_Config.h" and comment out - * "#define LOAD_SD_LIBRARY" and "#define LOAD_SDFAT_LIBRARY". - */ - -// Required by Arducam library -#include -#include -#include -// Arducam library -#include -// JPEGDecoder library -#include - -// Checks that the Arducam library has been correctly configured -#if !(defined OV2640_MINI_2MP_PLUS) -#error Please select the hardware platform and camera module in the Arduino/libraries/ArduCAM/memorysaver.h -#endif - -// The size of our temporary buffer for holding -// JPEG data received from the Arducam module -#define MAX_JPEG_BYTES 4096 -// The pin connected to the Arducam Chip Select -#define CS 7 - -// Camera library instance -ArduCAM myCAM(OV2640, CS); -// Temporary buffer for holding JPEG data from camera -uint8_t jpeg_buffer[MAX_JPEG_BYTES] = {0}; -// Length of the JPEG data currently in the buffer -uint32_t jpeg_length = 0; - -// Get the camera module ready -TfLiteStatus InitCamera(tflite::ErrorReporter* error_reporter) { - TF_LITE_REPORT_ERROR(error_reporter, "Attempting to start Arducam"); - // Enable the Wire library - Wire.begin(); - // Configure the CS pin - pinMode(CS, OUTPUT); - digitalWrite(CS, HIGH); - // initialize SPI - SPI.begin(); - // Reset the CPLD - myCAM.write_reg(0x07, 0x80); - delay(100); - myCAM.write_reg(0x07, 0x00); - delay(100); - // Test whether we can communicate with Arducam via SPI - myCAM.write_reg(ARDUCHIP_TEST1, 0x55); - uint8_t test; - test = myCAM.read_reg(ARDUCHIP_TEST1); - if (test != 0x55) { - TF_LITE_REPORT_ERROR(error_reporter, "Can't communicate with Arducam"); - delay(1000); - return kTfLiteError; - } - // Use JPEG capture mode, since it allows us to specify - // a resolution smaller than the full sensor frame - myCAM.set_format(JPEG); - myCAM.InitCAM(); - // Specify the smallest possible resolution - myCAM.OV2640_set_JPEG_size(OV2640_160x120); - delay(100); - return kTfLiteOk; -} - -// Begin the capture and wait for it to finish -TfLiteStatus PerformCapture(tflite::ErrorReporter* error_reporter) { - TF_LITE_REPORT_ERROR(error_reporter, "Starting capture"); - // Make sure the buffer is emptied before each capture - myCAM.flush_fifo(); - myCAM.clear_fifo_flag(); - // Start capture - myCAM.start_capture(); - // Wait for indication that it is done - while (!myCAM.get_bit(ARDUCHIP_TRIG, CAP_DONE_MASK)) { - } - TF_LITE_REPORT_ERROR(error_reporter, "Image captured"); - delay(50); - // Clear the capture done flag - myCAM.clear_fifo_flag(); - return kTfLiteOk; -} - -// Read data from the camera module into a local buffer -TfLiteStatus ReadData(tflite::ErrorReporter* error_reporter) { - // This represents the total length of the JPEG data - jpeg_length = myCAM.read_fifo_length(); - TF_LITE_REPORT_ERROR(error_reporter, "Reading %d bytes from Arducam", - jpeg_length); - // Ensure there's not too much data for our buffer - if (jpeg_length > MAX_JPEG_BYTES) { - TF_LITE_REPORT_ERROR(error_reporter, "Too many bytes in FIFO buffer (%d)", - MAX_JPEG_BYTES); - return kTfLiteError; - } - if (jpeg_length == 0) { - TF_LITE_REPORT_ERROR(error_reporter, "No data in Arducam FIFO buffer"); - return kTfLiteError; - } - myCAM.CS_LOW(); - myCAM.set_fifo_burst(); - for (int index = 0; index < jpeg_length; index++) { - jpeg_buffer[index] = SPI.transfer(0x00); - } - delayMicroseconds(15); - TF_LITE_REPORT_ERROR(error_reporter, "Finished reading"); - myCAM.CS_HIGH(); - return kTfLiteOk; -} - -// Decode the JPEG image, crop it, and convert it to greyscale -TfLiteStatus DecodeAndProcessImage(tflite::ErrorReporter* error_reporter, - int image_width, int image_height, - int8_t* image_data) { - TF_LITE_REPORT_ERROR(error_reporter, - "Decoding JPEG and converting to greyscale"); - // Parse the JPEG headers. The image will be decoded as a sequence of Minimum - // Coded Units (MCUs), which are 16x8 blocks of pixels. - JpegDec.decodeArray(jpeg_buffer, jpeg_length); - - // Crop the image by keeping a certain number of MCUs in each dimension - const int keep_x_mcus = image_width / JpegDec.MCUWidth; - const int keep_y_mcus = image_height / JpegDec.MCUHeight; - - // Calculate how many MCUs we will throw away on the x axis - const int skip_x_mcus = JpegDec.MCUSPerRow - keep_x_mcus; - // Roughly center the crop by skipping half the throwaway MCUs at the - // beginning of each row - const int skip_start_x_mcus = skip_x_mcus / 2; - // Index where we will start throwing away MCUs after the data - const int skip_end_x_mcu_index = skip_start_x_mcus + keep_x_mcus; - // Same approach for the columns - const int skip_y_mcus = JpegDec.MCUSPerCol - keep_y_mcus; - const int skip_start_y_mcus = skip_y_mcus / 2; - const int skip_end_y_mcu_index = skip_start_y_mcus + keep_y_mcus; - - // Pointer to the current pixel - uint16_t* pImg; - // Color of the current pixel - uint16_t color; - - // Loop over the MCUs - while (JpegDec.read()) { - // Skip over the initial set of rows - if (JpegDec.MCUy < skip_start_y_mcus) { - continue; - } - // Skip if we're on a column that we don't want - if (JpegDec.MCUx < skip_start_x_mcus || - JpegDec.MCUx >= skip_end_x_mcu_index) { - continue; - } - // Skip if we've got all the rows we want - if (JpegDec.MCUy >= skip_end_y_mcu_index) { - continue; - } - // Pointer to the current pixel - pImg = JpegDec.pImage; - - // The x and y indexes of the current MCU, ignoring the MCUs we skip - int relative_mcu_x = JpegDec.MCUx - skip_start_x_mcus; - int relative_mcu_y = JpegDec.MCUy - skip_start_y_mcus; - - // The coordinates of the top left of this MCU when applied to the output - // image - int x_origin = relative_mcu_x * JpegDec.MCUWidth; - int y_origin = relative_mcu_y * JpegDec.MCUHeight; - - // Loop through the MCU's rows and columns - for (int mcu_row = 0; mcu_row < JpegDec.MCUHeight; mcu_row++) { - // The y coordinate of this pixel in the output index - int current_y = y_origin + mcu_row; - for (int mcu_col = 0; mcu_col < JpegDec.MCUWidth; mcu_col++) { - // Read the color of the pixel as 16-bit integer - color = *pImg++; - // Extract the color values (5 red bits, 6 green, 5 blue) - uint8_t r, g, b; - r = ((color & 0xF800) >> 11) * 8; - g = ((color & 0x07E0) >> 5) * 4; - b = ((color & 0x001F) >> 0) * 8; - // Convert to grayscale by calculating luminance - // See https://en.wikipedia.org/wiki/Grayscale for magic numbers - float gray_value = (0.2126 * r) + (0.7152 * g) + (0.0722 * b); - - // Convert to signed 8-bit integer by subtracting 128. - gray_value -= 128; - - // The x coordinate of this pixel in the output image - int current_x = x_origin + mcu_col; - // The index of this pixel in our flat output buffer - int index = (current_y * image_width) + current_x; - image_data[index] = static_cast(gray_value); - } - } - } - TF_LITE_REPORT_ERROR(error_reporter, "Image decoded and processed"); - return kTfLiteOk; -} - -// Get an image from the camera module -TfLiteStatus GetImage(tflite::ErrorReporter* error_reporter, int image_width, - int image_height, int channels, int8_t* image_data) { - static bool g_is_camera_initialized = false; - if (!g_is_camera_initialized) { - TfLiteStatus init_status = InitCamera(error_reporter); - if (init_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter, "InitCamera failed"); - return init_status; - } - g_is_camera_initialized = true; - } - - TfLiteStatus capture_status = PerformCapture(error_reporter); - if (capture_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter, "PerformCapture failed"); - return capture_status; - } - - TfLiteStatus read_data_status = ReadData(error_reporter); - if (read_data_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter, "ReadData failed"); - return read_data_status; - } - - TfLiteStatus decode_status = DecodeAndProcessImage( - error_reporter, image_width, image_height, image_data); - if (decode_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter, "DecodeAndProcessImage failed"); - return decode_status; - } - - return kTfLiteOk; -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/arduino/main.cc b/tensorflow/lite/micro/examples/person_detection_experimental/arduino/main.cc deleted file mode 100644 index 89cbdccf3a5..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/arduino/main.cc +++ /dev/null @@ -1,20 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/main_functions.h" - -// Arduino automatically calls the setup() and loop() functions in a sketch, so -// where other systems need their own main routine in this file, it can be left -// empty. diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.cc b/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.cc deleted file mode 100644 index bc409b5b33a..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.cc +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h" - -// This dummy implementation writes person and no person scores to the error -// console. Real applications will want to take some custom action instead, and -// should implement their own versions of this function. -void RespondToDetection(tflite::ErrorReporter* error_reporter, - int8_t person_score, int8_t no_person_score) { - TF_LITE_REPORT_ERROR(error_reporter, "person score:%d no person score %d", - person_score, no_person_score); -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h b/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h deleted file mode 100644 index aadad3be9ef..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Provides an interface to take an action based on the output from the person -// detection model. - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_DETECTION_RESPONDER_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_DETECTION_RESPONDER_H_ - -#include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/micro/micro_error_reporter.h" - -// Called every time the results of a person detection run are available. The -// `person_score` has the numerical confidence that the captured image contains -// a person, and `no_person_score` has the numerical confidence that the image -// does not contain a person. Typically if person_score > no person score, the -// image is considered to contain a person. This threshold may be adjusted for -// particular applications. -void RespondToDetection(tflite::ErrorReporter* error_reporter, - int8_t person_score, int8_t no_person_score); - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_DETECTION_RESPONDER_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder_test.cc b/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder_test.cc deleted file mode 100644 index 2cf6f68d7f4..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/detection_responder_test.cc +++ /dev/null @@ -1,32 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h" - -#include "tensorflow/lite/micro/testing/micro_test.h" - -TF_LITE_MICRO_TESTS_BEGIN - -TF_LITE_MICRO_TEST(TestCallability) { - tflite::MicroErrorReporter micro_error_reporter; - - // This will have external side-effects (like printing to the debug console - // or lighting an LED) that are hard to observe, so the most we can do is - // make sure the call doesn't crash. - RespondToDetection(µ_error_reporter, -100, 100); - RespondToDetection(µ_error_reporter, 100, 50); -} - -TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.c b/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.c deleted file mode 100644 index 3ec481a5cd4..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.c +++ /dev/null @@ -1,719 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "HM01B0.h" - -#include "HM01B0_Walking1s_01.h" -#include "am_bsp.h" -#include "am_mcu_apollo.h" -#include "am_util.h" -#include "platform_Sparkfun_Edge.h" - -//#define ENABLE_ASYNC - -const am_hal_gpio_pincfg_t g_HM01B0_pin_vsync = { - .uFuncSel = 3, - .eGPOutcfg = AM_HAL_GPIO_PIN_OUTCFG_DISABLE, -#ifdef ENABLE_ASYNC - .eIntDir = AM_HAL_GPIO_PIN_INTDIR_BOTH, -#endif - .eGPInput = AM_HAL_GPIO_PIN_INPUT_ENABLE, - .eGPRdZero = AM_HAL_GPIO_PIN_RDZERO_READPIN}; - -const am_hal_gpio_pincfg_t g_HM01B0_pin_int = { - .uFuncSel = 3, - .eGPOutcfg = AM_HAL_GPIO_PIN_OUTCFG_DISABLE, - .eIntDir = AM_HAL_GPIO_PIN_INTDIR_LO2HI, - .eGPInput = AM_HAL_GPIO_PIN_INPUT_ENABLE, - .eGPRdZero = AM_HAL_GPIO_PIN_RDZERO_READPIN}; - -#ifdef ENABLE_ASYNC -static bool s_bVsyncAsserted = false; - -//***************************************************************************** -// -// GPIO ISR -// -//***************************************************************************** -static void hm01b0_gpio_isr(void) { - // - // Clear the GPIO Interrupt (write to clear). - // - am_hal_gpio_interrupt_clear(1 << HM01B0_PIN_VSYNC); - - if (read_vsync()) { - s_bVsyncAsserted = true; - } else { - s_bVsyncAsserted = false; - } -} -#endif - -//***************************************************************************** -// -//! @brief Write HM01B0 registers -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param ui16Reg - Register address. -//! @param pui8Value - Pointer to the data to be written. -//! @param ui32NumBytes - Length of the data in bytes to be written. -//! -//! This function writes value to HM01B0 registers. -//! -//! @return Error code. -// -//***************************************************************************** -static uint32_t hm01b0_write_reg(hm01b0_cfg_t* psCfg, uint16_t ui16Reg, - uint8_t* pui8Value, uint32_t ui32NumBytes) { - am_hal_iom_transfer_t Transaction; - - // - // Create the transaction. - // - Transaction.ui32InstrLen = sizeof(uint16_t); - Transaction.ui32Instr = (ui16Reg & 0x0000FFFF); - Transaction.eDirection = AM_HAL_IOM_TX; - Transaction.ui32NumBytes = ui32NumBytes; - Transaction.pui32TxBuffer = (uint32_t*)pui8Value; - Transaction.uPeerInfo.ui32I2CDevAddr = (uint32_t)psCfg->ui16SlvAddr; - Transaction.bContinue = false; - Transaction.ui8RepeatCount = 0; - Transaction.ui32PauseCondition = 0; - Transaction.ui32StatusSetClr = 0; - - // - // Execute the transction over IOM. - // - if (am_hal_iom_blocking_transfer(psCfg->pIOMHandle, &Transaction)) { - return HM01B0_ERR_I2C; - } - - return HM01B0_ERR_OK; -} - -//***************************************************************************** -// -//! @brief Read HM01B0 registers -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param ui16Reg - Register address. -//! @param pui8Value - Pointer to the buffer for read data to be put -//! into. -//! @param ui32NumBytes - Length of the data to be read. -//! -//! This function reads value from HM01B0 registers. -//! -//! @return Error code. -// -//***************************************************************************** -static uint32_t hm01b0_read_reg(hm01b0_cfg_t* psCfg, uint16_t ui16Reg, - uint8_t* pui8Value, uint32_t ui32NumBytes) { - am_hal_iom_transfer_t Transaction; - - // - // Create the transaction. - // - Transaction.ui32InstrLen = sizeof(uint16_t); - Transaction.ui32Instr = (ui16Reg & 0x0000FFFF); - Transaction.eDirection = AM_HAL_IOM_RX; - Transaction.ui32NumBytes = ui32NumBytes; - Transaction.pui32RxBuffer = (uint32_t*)pui8Value; - ; - Transaction.uPeerInfo.ui32I2CDevAddr = (uint32_t)psCfg->ui16SlvAddr; - Transaction.bContinue = false; - Transaction.ui8RepeatCount = 0; - Transaction.ui32PauseCondition = 0; - Transaction.ui32StatusSetClr = 0; - - // - // Execute the transction over IOM. - // - if (am_hal_iom_blocking_transfer(psCfg->pIOMHandle, &Transaction)) { - return HM01B0_ERR_I2C; - } - - return HM01B0_ERR_OK; -} - -//***************************************************************************** -// -//! @brief Load HM01B0 a given script -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param psScrip - Pointer to the script to be loaded. -//! @param ui32ScriptCmdNum - Number of entries in a given script. -//! -//! This function loads HM01B0 a given script. -//! -//! @return Error code. -// -//***************************************************************************** -static uint32_t hm01b0_load_script(hm01b0_cfg_t* psCfg, hm_script_t* psScript, - uint32_t ui32ScriptCmdNum) { - uint32_t ui32Err = HM01B0_ERR_OK; - for (uint32_t idx = 0; idx < ui32ScriptCmdNum; idx++) { - ui32Err = hm01b0_write_reg(psCfg, (psScript + idx)->ui16Reg, - &((psScript + idx)->ui8Val), sizeof(uint8_t)); - if (ui32Err != HM01B0_ERR_OK) { - break; - } - } - - return ui32Err; -} - -//***************************************************************************** -// -//! @brief Power up HM01B0 -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function powers up HM01B0. -//! -//! @return none. -// -//***************************************************************************** -void hm01b0_power_up(hm01b0_cfg_t* psCfg) { - // place holder -} - -//***************************************************************************** -// -//! @brief Power down HM01B0 -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function powers up HM01B0. -//! -//! @return none. -// -//***************************************************************************** -void hm01b0_power_down(hm01b0_cfg_t* psCfg) { - // place holder -} - -//***************************************************************************** -// -//! @brief Enable MCLK -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function utilizes CTimer to generate MCLK for HM01B0. -//! -//! @return none. -// -//***************************************************************************** -void hm01b0_mclk_enable(hm01b0_cfg_t* psCfg) { -#define MCLK_UI64PATTERN 0x55555555 -#define MCLK_UI64PATTERNLEN 31 - - am_hal_clkgen_control(AM_HAL_CLKGEN_CONTROL_SYSCLK_MAX, 0); - - // - // Set up timer. - // - am_hal_ctimer_clear(psCfg->ui32CTimerModule, psCfg->ui32CTimerSegment); - - am_hal_ctimer_config_single( - psCfg->ui32CTimerModule, psCfg->ui32CTimerSegment, - (AM_HAL_CTIMER_FN_PTN_REPEAT | AM_HAL_CTIMER_HFRC_12MHZ)); - - // - // Set the pattern in the CMPR registers. - // - am_hal_ctimer_compare_set(psCfg->ui32CTimerModule, psCfg->ui32CTimerSegment, - 0, (uint32_t)(MCLK_UI64PATTERN & 0xFFFF)); - am_hal_ctimer_compare_set(psCfg->ui32CTimerModule, psCfg->ui32CTimerSegment, - 1, (uint32_t)((MCLK_UI64PATTERN >> 16) & 0xFFFF)); - - // - // Set the timer trigger and pattern length. - // - am_hal_ctimer_config_trigger( - psCfg->ui32CTimerModule, psCfg->ui32CTimerSegment, - ((MCLK_UI64PATTERNLEN << CTIMER_AUX0_TMRA0LMT_Pos) | - (CTIMER_AUX0_TMRB0TRIG_DIS << CTIMER_AUX0_TMRA0TRIG_Pos))); - - // - // Configure timer output pin. - // - am_hal_ctimer_output_config(psCfg->ui32CTimerModule, psCfg->ui32CTimerSegment, - psCfg->ui32CTimerOutputPin, - AM_HAL_CTIMER_OUTPUT_NORMAL, - AM_HAL_GPIO_PIN_DRIVESTRENGTH_12MA); - - // - // Start the timer. - // - am_hal_ctimer_start(psCfg->ui32CTimerModule, psCfg->ui32CTimerSegment); -} - -//***************************************************************************** -// -//! @brief Disable MCLK -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function disable CTimer to stop MCLK for HM01B0. -//! -//! @return none. -// -//***************************************************************************** -void hm01b0_mclk_disable(hm01b0_cfg_t* psCfg) { - // - // Stop the timer. - // - am_hal_ctimer_stop(psCfg->ui32CTimerModule, psCfg->ui32CTimerSegment); - am_hal_gpio_pinconfig(psCfg->ui32CTimerOutputPin, g_AM_HAL_GPIO_DISABLE); -} - -//***************************************************************************** -// -//! @brief Initialize interfaces -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function initializes interfaces. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_init_if(hm01b0_cfg_t* psCfg) { - void* pIOMHandle = NULL; - - if (psCfg->ui32IOMModule > AM_REG_IOM_NUM_MODULES) { - return HM01B0_ERR_I2C; - } - - // - // Enable fault detection. - // -#if AM_APOLLO3_MCUCTRL - am_hal_mcuctrl_control(AM_HAL_MCUCTRL_CONTROL_FAULT_CAPTURE_ENABLE, 0); -#else // AM_APOLLO3_MCUCTRL - am_hal_mcuctrl_fault_capture_enable(); -#endif // AM_APOLLO3_MCUCTRL - - // - // Initialize the IOM instance. - // Enable power to the IOM instance. - // Configure the IOM for Serial operation during initialization. - // Enable the IOM. - // - if (am_hal_iom_initialize(psCfg->ui32IOMModule, &pIOMHandle) || - am_hal_iom_power_ctrl(pIOMHandle, AM_HAL_SYSCTRL_WAKE, false) || - am_hal_iom_configure(pIOMHandle, &(psCfg->sIOMCfg)) || - am_hal_iom_enable(pIOMHandle)) { - return HM01B0_ERR_I2C; - } else { - // - // Configure the IOM pins. - // - am_bsp_iom_pins_enable(psCfg->ui32IOMModule, psCfg->eIOMMode); - - psCfg->pIOMHandle = pIOMHandle; - } - - // initialize pins for camera parallel interface. - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD0); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD1); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD2); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD3); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD4); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD5); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD6); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD7); - - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD0); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD1); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD2); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD3); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD4); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD5); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD6); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD7); - - am_hal_gpio_fast_pinconfig( - (uint64_t)0x1 << psCfg->ui8PinD0 | (uint64_t)0x1 << psCfg->ui8PinD1 | - (uint64_t)0x1 << psCfg->ui8PinD2 | (uint64_t)0x1 << psCfg->ui8PinD3 | - (uint64_t)0x1 << psCfg->ui8PinD4 | (uint64_t)0x1 << psCfg->ui8PinD5 | - (uint64_t)0x1 << psCfg->ui8PinD6 | (uint64_t)0x1 << psCfg->ui8PinD7, - g_AM_HAL_GPIO_INPUT, 0); - - am_hal_gpio_pinconfig(psCfg->ui8PinVSYNC, g_HM01B0_pin_vsync); -#ifdef ENABLE_ASYNC - psCfg->pfnGpioIsr = hm01b0_gpio_isr; - am_hal_gpio_interrupt_clear(AM_HAL_GPIO_BIT(psCfg->ui8PinVSYNC)); - am_hal_gpio_interrupt_enable(AM_HAL_GPIO_BIT(psCfg->ui8PinVSYNC)); - NVIC_EnableIRQ(GPIO_IRQn); -#endif - am_hal_gpio_pinconfig(psCfg->ui8PinHSYNC, g_AM_HAL_GPIO_INPUT); - am_hal_gpio_pinconfig(psCfg->ui8PinPCLK, g_AM_HAL_GPIO_INPUT); - - am_hal_gpio_pinconfig(psCfg->ui8PinTrig, g_AM_HAL_GPIO_OUTPUT); - - am_hal_gpio_pinconfig(psCfg->ui8PinInt, g_AM_HAL_GPIO_DISABLE); - // am_hal_gpio_pinconfig(psCfg->ui8PinInt, g_HM01B0_pin_int); - // am_hal_gpio_interrupt_clear(AM_HAL_GPIO_BIT(psCfg->ui8PinInt)); - // am_hal_gpio_interrupt_enable(AM_HAL_GPIO_BIT(psCfg->ui8PinInt)); - // NVIC_EnableIRQ(GPIO_IRQn); - - return HM01B0_ERR_OK; -} - -//***************************************************************************** -// -//! @brief Deinitialize interfaces -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function deinitializes interfaces. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_deinit_if(hm01b0_cfg_t* psCfg) { - am_hal_iom_disable(psCfg->pIOMHandle); - am_hal_iom_uninitialize(psCfg->pIOMHandle); - - am_hal_gpio_pinconfig(psCfg->ui8PinSCL, g_AM_HAL_GPIO_DISABLE); - am_hal_gpio_pinconfig(psCfg->ui8PinSDA, g_AM_HAL_GPIO_DISABLE); - - // initialize pins for camera parallel interface. - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD0); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD1); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD2); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD3); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD4); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD5); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD6); - am_hal_gpio_fastgpio_disable(psCfg->ui8PinD7); - - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD0); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD1); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD2); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD3); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD4); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD5); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD6); - am_hal_gpio_fastgpio_clr(psCfg->ui8PinD7); - - am_hal_gpio_pinconfig(psCfg->ui8PinVSYNC, g_AM_HAL_GPIO_DISABLE); -#ifdef ENABLE_ASYNC - NVIC_DisableIRQ(GPIO_IRQn); - am_hal_gpio_interrupt_disable(AM_HAL_GPIO_BIT(psCfg->ui8PinVSYNC)); - am_hal_gpio_interrupt_clear(AM_HAL_GPIO_BIT(psCfg->ui8PinVSYNC)); - psCfg->pfnGpioIsr = NULL; -#endif - am_hal_gpio_pinconfig(psCfg->ui8PinHSYNC, g_AM_HAL_GPIO_DISABLE); - am_hal_gpio_pinconfig(psCfg->ui8PinPCLK, g_AM_HAL_GPIO_DISABLE); - - am_hal_gpio_pinconfig(psCfg->ui8PinTrig, g_AM_HAL_GPIO_DISABLE); - am_hal_gpio_pinconfig(psCfg->ui8PinInt, g_AM_HAL_GPIO_DISABLE); - - return HM01B0_ERR_OK; -} - -//***************************************************************************** -// -//! @brief Get HM01B0 Model ID -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param pui16MID - Pointer to buffer for the read back model ID. -//! -//! This function reads back HM01B0 model ID. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_get_modelid(hm01b0_cfg_t* psCfg, uint16_t* pui16MID) { - uint8_t ui8Data[1]; - uint32_t ui32Err; - - *pui16MID = 0x0000; - - ui32Err = - hm01b0_read_reg(psCfg, HM01B0_REG_MODEL_ID_H, ui8Data, sizeof(ui8Data)); - if (ui32Err == HM01B0_ERR_OK) { - *pui16MID |= (ui8Data[0] << 8); - } - - ui32Err = - hm01b0_read_reg(psCfg, HM01B0_REG_MODEL_ID_L, ui8Data, sizeof(ui8Data)); - if (ui32Err == HM01B0_ERR_OK) { - *pui16MID |= ui8Data[0]; - } - - return ui32Err; -} - -//***************************************************************************** -// -//! @brief Initialize HM01B0 -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param psScript - Pointer to HM01B0 initialization script. -//! @param ui32ScriptCmdNum - No. of commands in HM01B0 initialization -//! script. -//! -//! This function initilizes HM01B0 with a given script. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_init_system(hm01b0_cfg_t* psCfg, hm_script_t* psScript, - uint32_t ui32ScriptCmdNum) { - return hm01b0_load_script(psCfg, psScript, ui32ScriptCmdNum); -} - -//***************************************************************************** -// -//! @brief Set HM01B0 in the walking 1s test mode -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function sets HM01B0 in the walking 1s test mode. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_test_walking1s(hm01b0_cfg_t* psCfg) { - uint32_t ui32ScriptCmdNum = - sizeof(sHM01b0TestModeScript_Walking1s) / sizeof(hm_script_t); - hm_script_t* psScript = (hm_script_t*)sHM01b0TestModeScript_Walking1s; - - return hm01b0_load_script(psCfg, psScript, ui32ScriptCmdNum); -} - -//***************************************************************************** -// -//! @brief Software reset HM01B0 -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function resets HM01B0 by issuing a reset command. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_reset_sw(hm01b0_cfg_t* psCfg) { - uint8_t ui8Data[1] = {0x00}; - return hm01b0_write_reg(psCfg, HM01B0_REG_SW_RESET, ui8Data, sizeof(ui8Data)); -} - -//***************************************************************************** -// -//! @brief Get current HM01B0 operation mode. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param pui8Mode - Pointer to buffer -//! - for the read back operation mode to be put into -//! -//! This function get HM01B0 operation mode. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_get_mode(hm01b0_cfg_t* psCfg, uint8_t* pui8Mode) { - uint8_t ui8Data[1] = {0x01}; - uint32_t ui32Err; - - ui32Err = - hm01b0_read_reg(psCfg, HM01B0_REG_MODE_SELECT, ui8Data, sizeof(ui8Data)); - - *pui8Mode = ui8Data[0]; - - return ui32Err; -} - -//***************************************************************************** -// -//! @brief Set HM01B0 operation mode. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param ui8Mode - Operation mode. One of: -//! HM01B0_REG_MODE_SELECT_STANDBY -//! HM01B0_REG_MODE_SELECT_STREAMING -//! HM01B0_REG_MODE_SELECT_STREAMING_NFRAMES -//! HM01B0_REG_MODE_SELECT_STREAMING_HW_TRIGGER -//! @param ui8FrameCnt - Frame count for -//! HM01B0_REG_MODE_SELECT_STREAMING_NFRAMES. -//! - Discarded if other modes. -//! -//! This function set HM01B0 operation mode. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_set_mode(hm01b0_cfg_t* psCfg, uint8_t ui8Mode, - uint8_t ui8FrameCnt) { - uint32_t ui32Err = HM01B0_ERR_OK; - - if (ui8Mode == HM01B0_REG_MODE_SELECT_STREAMING_NFRAMES) { - ui32Err = hm01b0_write_reg(psCfg, HM01B0_REG_PMU_PROGRAMMABLE_FRAMECNT, - &ui8FrameCnt, sizeof(ui8FrameCnt)); - } - - if (ui32Err == HM01B0_ERR_OK) { - ui32Err = hm01b0_write_reg(psCfg, HM01B0_REG_MODE_SELECT, &ui8Mode, - sizeof(ui8Mode)); - } - - return ui32Err; -} - -//***************************************************************************** -// -//! @brief Hardware trigger HM01B0 to stream. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param bTrigger - True to start streaming -//! - False to stop streaming -//! -//! This function triggers HM01B0 to stream by toggling the TRIG pin. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_hardware_trigger_streaming(hm01b0_cfg_t* psCfg, bool bTrigger) { - uint32_t ui32Err = HM01B0_ERR_OK; - uint8_t ui8Mode; - - ui32Err = hm01b0_get_mode(psCfg, &ui8Mode); - - if (ui32Err != HM01B0_ERR_OK) goto end; - - if (ui8Mode != HM01B0_REG_MODE_SELECT_STREAMING_HW_TRIGGER) { - ui32Err = HM01B0_ERR_MODE; - goto end; - } - - if (bTrigger) { - am_hal_gpio_output_set(psCfg->ui8PinTrig); - } else { - am_hal_gpio_output_clear(psCfg->ui8PinTrig); - } - -end: - return ui32Err; -} - -//***************************************************************************** -// -//! @brief Set HM01B0 mirror mode. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param bHmirror - Horizontal mirror -//! @param bVmirror - Vertical mirror -//! -//! This function set HM01B0 mirror mode. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_set_mirror(hm01b0_cfg_t* psCfg, bool bHmirror, bool bVmirror) { - uint8_t ui8Data = 0x00; - uint32_t ui32Err = HM01B0_ERR_OK; - - if (bHmirror) { - ui8Data |= HM01B0_REG_IMAGE_ORIENTATION_HMIRROR; - } - - if (bVmirror) { - ui8Data |= HM01B0_REG_IMAGE_ORIENTATION_VMIRROR; - } - - ui32Err = hm01b0_write_reg(psCfg, HM01B0_REG_IMAGE_ORIENTATION, &ui8Data, - sizeof(ui8Data)); - - if (ui32Err == HM01B0_ERR_OK) { - ui8Data = HM01B0_REG_GRP_PARAM_HOLD_HOLD; - ui32Err = hm01b0_write_reg(psCfg, HM01B0_REG_GRP_PARAM_HOLD, &ui8Data, - sizeof(ui8Data)); - } - - return ui32Err; -} - -//***************************************************************************** -// -//! @brief Read data of one frame from HM01B0. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param pui8Buffer - Pointer to the frame buffer. -//! @param ui32BufferLen - Framebuffer size. -//! -//! This function read data of one frame from HM01B0. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_blocking_read_oneframe(hm01b0_cfg_t* psCfg, uint8_t* pui8Buffer, - uint32_t ui32BufferLen) { - uint32_t ui32Err = HM01B0_ERR_OK; - uint32_t ui32Idx = 0x00; - - am_util_stdio_printf("[%s] +\n", __func__); -#ifdef ENABLE_ASYNC - while (!s_bVsyncAsserted); - - while (s_bVsyncAsserted) { - // we don't check HSYNC here on the basis of assuming HM01B0 in the gated - // PCLK mode which PCLK toggles only when HSYNC is asserted. And also to - // minimize the overhead of polling. - - if (read_pclk()) { - *(pui8Buffer + ui32Idx++) = read_byte(); - - if (ui32Idx == ui32BufferLen) { - goto end; - } - - while (read_pclk()); - } - } -#else - uint32_t ui32HsyncCnt = 0x00; - - while ((ui32HsyncCnt < HM01B0_PIXEL_Y_NUM)) { - while (0x00 == read_hsync()); - - // read one row - while (read_hsync()) { - while (0x00 == read_pclk()); - - *(pui8Buffer + ui32Idx++) = read_byte(); - - if (ui32Idx == ui32BufferLen) { - goto end; - } - - while (read_pclk()); - } - - ui32HsyncCnt++; - } -#endif -end: - am_util_stdio_printf("[%s] - Byte Counts %d\n", __func__, ui32Idx); - return ui32Err; -} - -uint32_t hm01b0_single_frame_capture(hm01b0_cfg_t* psCfg) { - hm01b0_write_reg(psCfg, HM01B0_REG_PMU_PROGRAMMABLE_FRAMECNT, 0x01, 1); - hm01b0_write_reg(psCfg, HM01B0_REG_MODE_SELECT, - HM01B0_REG_MODE_SELECT_STREAMING_NFRAMES, 1); - hm01b0_write_reg(psCfg, HM01B0_REG_GRP_PARAM_HOLD, 0x01, 1); -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.h b/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.h deleted file mode 100644 index f95ee7bd76c..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.h +++ /dev/null @@ -1,402 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_H_ - -#ifdef __cplusplus -extern "C" { -#endif -#include "am_bsp.h" // NOLINT -#include "am_mcu_apollo.h" // NOLINT -#include "am_util.h" // NOLINT - -#define HM01B0_DRV_VERSION (0) -#define HM01B0_DRV_SUBVERSION (3) - -#define HM01B0_DEFAULT_ADDRESS (0x24) - -#define HM01B0_PIXEL_X_NUM (324) -#define HM01B0_PIXEL_Y_NUM (244) - -#define HM01B0_REG_MODEL_ID_H (0x0000) -#define HM01B0_REG_MODEL_ID_L (0x0001) -#define HM01B0_REG_SILICON_REV (0x0002) -#define HM01B0_REG_FRAME_COUNT (0x0005) -#define HM01B0_REG_PIXEL_ORDER (0x0006) - -#define HM01B0_REG_MODE_SELECT (0x0100) -#define HM01B0_REG_IMAGE_ORIENTATION (0x0101) -#define HM01B0_REG_SW_RESET (0x0103) -#define HM01B0_REG_GRP_PARAM_HOLD (0x0104) - -#define HM01B0_REG_I2C_ID_SEL (0x3400) -#define HM01B0_REG_I2C_ID_REG (0x3401) - -#define HM01B0_REG_PMU_PROGRAMMABLE_FRAMECNT (0x3020) - -// #define HM01B0_REG_MODE_SELECT (0x0100) -#define HM01B0_REG_MODE_SELECT_STANDBY (0x00) -#define HM01B0_REG_MODE_SELECT_STREAMING (0x01) -#define HM01B0_REG_MODE_SELECT_STREAMING_NFRAMES (0x03) -#define HM01B0_REG_MODE_SELECT_STREAMING_HW_TRIGGER (0x05) - -// #define HM01B0_REG_IMAGE_ORIENTATION (0x0101) -#define HM01B0_REG_IMAGE_ORIENTATION_DEFAULT (0x00) -#define HM01B0_REG_IMAGE_ORIENTATION_HMIRROR (0x01) -#define HM01B0_REG_IMAGE_ORIENTATION_VMIRROR (0x02) -#define HM01B0_REG_IMAGE_ORIENTATION_HVMIRROR \ - (HM01B0_REG_IMAGE_ORIENTATION_HMIRROR | HM01B0_REG_IMAGE_ORIENTATION_HVMIRROR) - -// #define HM01B0_REG_GRP_PARAM_HOLD (0x0104) -#define HM01B0_REG_GRP_PARAM_HOLD_CONSUME (0x00) -#define HM01B0_REG_GRP_PARAM_HOLD_HOLD (0x01) - -// Helpers for reading raw values from the camera. -#define read_vsync() \ - (AM_REGVAL(AM_REGADDR(GPIO, RDA)) & (1 << HM01B0_PIN_VSYNC)) -#define read_hsync() \ - (AM_REGVAL(AM_REGADDR(GPIO, RDA)) & (1 << HM01B0_PIN_HSYNC)) -#define read_pclk() (AM_REGVAL(AM_REGADDR(GPIO, RDA)) & (1 << HM01B0_PIN_PCLK)) -#define read_byte() (APBDMA->BBINPUT) - -enum { - HM01B0_ERR_OK = 0x00, - HM01B0_ERR_I2C, - HM01B0_ERR_MODE, -}; - -typedef struct { - uint16_t ui16Reg; - uint8_t ui8Val; -} hm_script_t; - -typedef struct { - uint16_t ui16SlvAddr; - am_hal_iom_mode_e eIOMMode; - uint32_t ui32IOMModule; - am_hal_iom_config_t sIOMCfg; - void *pIOMHandle; - - uint32_t ui32CTimerModule; - uint32_t ui32CTimerSegment; - uint32_t ui32CTimerOutputPin; - - uint8_t ui8PinSCL; - uint8_t ui8PinSDA; - uint8_t ui8PinD0; - uint8_t ui8PinD1; - uint8_t ui8PinD2; - uint8_t ui8PinD3; - uint8_t ui8PinD4; - uint8_t ui8PinD5; - uint8_t ui8PinD6; - uint8_t ui8PinD7; - uint8_t ui8PinVSYNC; - uint8_t ui8PinHSYNC; - uint8_t ui8PinPCLK; - - uint8_t ui8PinTrig; - uint8_t ui8PinInt; - void (*pfnGpioIsr)(void); -} hm01b0_cfg_t; - -//***************************************************************************** -// -//! @brief Write HM01B0 registers -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param ui16Reg - Register address. -//! @param pui8Value - Pointer to the data to be written. -//! @param ui32NumBytes - Length of the data in bytes to be written. -//! -//! This function writes value to HM01B0 registers. -//! -//! @return Error code. -// -//***************************************************************************** -static uint32_t hm01b0_write_reg(hm01b0_cfg_t *psCfg, uint16_t ui16Reg, - uint8_t *pui8Value, uint32_t ui32NumBytes); - -//***************************************************************************** -// -//! @brief Read HM01B0 registers -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param ui16Reg - Register address. -//! @param pui8Value - Pointer to the buffer for read data to be put -//! into. -//! @param ui32NumBytes - Length of the data to be read. -//! -//! This function reads value from HM01B0 registers. -//! -//! @return Error code. -// -//***************************************************************************** -static uint32_t hm01b0_read_reg(hm01b0_cfg_t *psCfg, uint16_t ui16Reg, - uint8_t *pui8Value, uint32_t ui32NumBytes); - -//***************************************************************************** -// -//! @brief Load HM01B0 a given script -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param psScrip - Pointer to the script to be loaded. -//! @param ui32ScriptCmdNum - Number of entries in a given script. -//! -//! This function loads HM01B0 a given script. -//! -//! @return Error code. -// -//***************************************************************************** -static uint32_t hm01b0_load_script(hm01b0_cfg_t *psCfg, hm_script_t *psScript, - uint32_t ui32ScriptCmdNum); - -//***************************************************************************** -// -//! @brief Power up HM01B0 -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function powers up HM01B0. -//! -//! @return none. -// -//***************************************************************************** -void hm01b0_power_up(hm01b0_cfg_t *psCfg); - -//***************************************************************************** -// -//! @brief Power down HM01B0 -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function powers up HM01B0. -//! -//! @return none. -// -//***************************************************************************** -void hm01b0_power_down(hm01b0_cfg_t *psCfg); - -//***************************************************************************** -// -//! @brief Enable MCLK -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function utilizes CTimer to generate MCLK for HM01B0. -//! -//! @return none. -// -//***************************************************************************** -void hm01b0_mclk_enable(hm01b0_cfg_t *psCfg); - -//***************************************************************************** -// -//! @brief Disable MCLK -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function disable CTimer to stop MCLK for HM01B0. -//! -//! @return none. -// -//***************************************************************************** -void hm01b0_mclk_disable(hm01b0_cfg_t *psCfg); - -//***************************************************************************** -// -//! @brief Initialize interfaces -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function initializes interfaces. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_init_if(hm01b0_cfg_t *psCfg); - -//***************************************************************************** -// -//! @brief Deinitialize interfaces -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function deinitializes interfaces. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_deinit_if(hm01b0_cfg_t *psCfg); - -//***************************************************************************** -// -//! @brief Get HM01B0 Model ID -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param pui16MID - Pointer to buffer for the read back model ID. -//! -//! This function reads back HM01B0 model ID. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_get_modelid(hm01b0_cfg_t *psCfg, uint16_t *pui16MID); - -//***************************************************************************** -// -//! @brief Initialize HM01B0 -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param psScript - Pointer to HM01B0 initialization script. -//! @param ui32ScriptCmdNum - No. of commands in HM01B0 initialization -//! script. -//! -//! This function initilizes HM01B0 with a given script. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_init_system(hm01b0_cfg_t *psCfg, hm_script_t *psScript, - uint32_t ui32ScriptCmdNum); - -//***************************************************************************** -// -//! @brief Set HM01B0 in the walking 1s test mode -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function sets HM01B0 in the walking 1s test mode. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_test_walking1s(hm01b0_cfg_t *psCfg); - -//***************************************************************************** -// -//! @brief Software reset HM01B0 -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function resets HM01B0 by issuing a reset command. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_reset_sw(hm01b0_cfg_t *psCfg); - -//***************************************************************************** -// -//! @brief Get current HM01B0 operation mode. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param pui8Mode - Pointer to buffer -//! - for the read back operation mode to be put into -//! -//! This function get HM01B0 operation mode. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_get_mode(hm01b0_cfg_t *psCfg, uint8_t *pui8Mode); - -//***************************************************************************** -// -//! @brief Set HM01B0 operation mode. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param ui8Mode - Operation mode. One of: -//! HM01B0_REG_MODE_SELECT_STANDBY -//! HM01B0_REG_MODE_SELECT_STREAMING -//! HM01B0_REG_MODE_SELECT_STREAMING_NFRAMES -//! HM01B0_REG_MODE_SELECT_STREAMING_HW_TRIGGER -//! @param framecnt - Frame count for -//! HM01B0_REG_MODE_SELECT_STREAMING_NFRAMES. -//! - Discarded if other modes. -//! -//! This function set HM01B0 operation mode. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_set_mode(hm01b0_cfg_t *psCfg, uint8_t ui8Mode, - uint8_t framecnt); - -//***************************************************************************** -// -//! @brief Hardware trigger HM01B0 to stream. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param bTrigger - True to start streaming -//! - False to stop streaming -//! -//! This function triggers HM01B0 to stream by toggling the TRIG pin. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_hardware_trigger_streaming(hm01b0_cfg_t *psCfg, bool bTrigger); - -//***************************************************************************** -// -//! @brief Set HM01B0 mirror mode. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param bHmirror - Horizontal mirror -//! @param bVmirror - Vertical mirror -//! -//! This function set HM01B0 mirror mode. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_set_mirror(hm01b0_cfg_t *psCfg, bool bHmirror, bool bVmirror); - -//***************************************************************************** -// -//! @brief Read data of one frame from HM01B0. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! @param pui8Buffer - Pointer to the frame buffer. -//! @param ui32BufferLen - Framebuffer size. -//! -//! This function read data of one frame from HM01B0. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_blocking_read_oneframe(hm01b0_cfg_t *psCfg, uint8_t *pui8Buffer, - uint32_t ui32BufferLen); - -//***************************************************************************** -// -//! @brief Read data of one frame from HM01B0. -//! -//! @param psCfg - Pointer to HM01B0 configuration structure. -//! -//! This function wakes up the camera and captures a single frame. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_single_frame_capture(hm01b0_cfg_t *psCfg); - -#ifdef __cplusplus -} -#endif - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_RAW8_QVGA_8bits_lsb_5fps.h b/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_RAW8_QVGA_8bits_lsb_5fps.h deleted file mode 100644 index ae78ca86c5f..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_RAW8_QVGA_8bits_lsb_5fps.h +++ /dev/null @@ -1,510 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_RAW8_QVGA_8BITS_LSB_5FPS_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_RAW8_QVGA_8BITS_LSB_5FPS_H_ - -#include "HM01B0.h" - -const hm_script_t sHM01B0InitScript[] = { - // ;************************************************************************* - // ; Sensor: HM01B0 - // ; I2C ID: 24 - // ; Resolution: 324x244 - // ; Lens: - // ; Flicker: - // ; Frequency: - // ; Description: AE control enable - // ; 8-bit mode, LSB first - // ; - // ; - // ; Note: - // ; - // ; $Revision: 1338 $ - // ; $Date:: 2017-04-11 15:43:45 +0800#$ - // ;************************************************************************* - // - // // --------------------------------------------------- - // // HUB system initial - // // --------------------------------------------------- - // W 20 8A04 01 2 1 - // W 20 8A00 22 2 1 - // W 20 8A01 00 2 1 - // W 20 8A02 01 2 1 - // W 20 0035 93 2 1 ; [3]&[1] hub616 20bits in, [5:4]=1 mclk=48/2=24mhz - // W 20 0036 00 2 1 - // W 20 0011 09 2 1 - // W 20 0012 B6 2 1 - // W 20 0014 08 2 1 - // W 20 0015 98 2 1 - // ;W 20 0130 16 2 1 ; 3m soc, signal buffer control - // ;W 20 0100 44 2 1 ; [6] hub616 20bits in - // W 20 0100 04 2 1 ; [6] hub616 20bits in - // W 20 0121 01 2 1 ; [0] Q1 Intf enable, [1]:4bit mode, [2] msb first, [3] - // serial mode - // W 20 0150 00 2 1 ; - // W 20 0150 04 2 1 ; - // - // - // //--------------------------------------------------- - // // Initial - // //--------------------------------------------------- - // W 24 0103 00 2 1 ; software reset-> was 0x22 - { - 0x0103, - 0x00, - }, - // W 24 0100 00 2 1; power up - { - 0x0100, - 0x00, - }, - // - // - // - // //--------------------------------------------------- - // // Analog - // //--------------------------------------------------- - // L HM01B0_analog_setting.txt - { - 0x1003, - 0x08, - }, - { - 0x1007, - 0x08, - }, - { - 0x3044, - 0x0A, - }, - { - 0x3045, - 0x00, - }, - { - 0x3047, - 0x0A, - }, - { - 0x3050, - 0xC0, - }, - { - 0x3051, - 0x42, - }, - { - 0x3052, - 0x50, - }, - { - 0x3053, - 0x00, - }, - { - 0x3054, - 0x03, - }, - { - 0x3055, - 0xF7, - }, - { - 0x3056, - 0xF8, - }, - { - 0x3057, - 0x29, - }, - { - 0x3058, - 0x1F, - }, - { - 0x3059, - 0x1E, - }, - { - 0x3064, - 0x00, - }, - { - 0x3065, - 0x04, - }, - // - // - // //--------------------------------------------------- - // // Digital function - // //--------------------------------------------------- - // - // // BLC - // W 24 1000 43 2 1 ; BLC_on, IIR - { - 0x1000, - 0x43, - }, - // W 24 1001 40 2 1 ; [6] : BLC dithering en - { - 0x1001, - 0x40, - }, - // W 24 1002 32 2 1 ; // blc_darkpixel_thd - { - 0x1002, - 0x32, - }, - // - // // Dgain - // W 24 0350 7F 2 1 ; Dgain Control - { - 0x0350, - 0x7F, - }, - // - // // BLI - // W 24 1006 01 2 1 ; [0] : bli enable - { - 0x1006, - 0x01, - }, - // - // // DPC - // W 24 1008 00 2 1 ; [2:0] : DPC option 0: DPC off 1 : mono 3 : bayer1 5 : - // bayer2 - { - 0x1008, - 0x00, - }, - // W 24 1009 A0 2 1 ; cluster hot pixel th - { - 0x1009, - 0xA0, - }, - // W 24 100A 60 2 1 ; cluster cold pixel th - { - 0x100A, - 0x60, - }, - // W 24 100B 90 2 1 ; single hot pixel th - { - 0x100B, - 0x90, - }, - // W 24 100C 40 2 1 ; single cold pixel th - { - 0x100C, - 0x40, - }, - // // - // advance VSYNC by 1 row - { - 0x3022, - 0x01, - }, - // W 24 1012 00 2 1 ; Sync. enable VSYNC shift - { - 0x1012, - 0x01, - }, - - // - // // ROI Statistic - // W 24 2000 07 2 1 ; [0] : AE stat en [1] : MD LROI stat en [2] : MD GROI - // stat en [3] : RGB stat ratio en [4] : IIR selection (1 -> 16, 0 -> 8) - { - 0x2000, - 0x07, - }, - // W 24 2003 00 2 1 ; MD GROI 0 y start HB - { - 0x2003, - 0x00, - }, - // W 24 2004 1C 2 1 ; MD GROI 0 y start LB - { - 0x2004, - 0x1C, - }, - // W 24 2007 00 2 1 ; MD GROI 1 y start HB - { - 0x2007, - 0x00, - }, - // W 24 2008 58 2 1 ; MD GROI 1 y start LB - { - 0x2008, - 0x58, - }, - // W 24 200B 00 2 1 ; MD GROI 2 y start HB - { - 0x200B, - 0x00, - }, - // W 24 200C 7A 2 1 ; MD GROI 2 y start LB - { - 0x200C, - 0x7A, - }, - // W 24 200F 00 2 1 ; MD GROI 3 y start HB - { - 0x200F, - 0x00, - }, - // W 24 2010 B8 2 1 ; MD GROI 3 y start LB - { - 0x2010, - 0xB8, - }, - // - // W 24 2013 00 2 1 ; MD LRIO y start HB - { - 0x2013, - 0x00, - }, - // W 24 2014 58 2 1 ; MD LROI y start LB - { - 0x2014, - 0x58, - }, - // W 24 2017 00 2 1 ; MD LROI y end HB - { - 0x2017, - 0x00, - }, - // W 24 2018 9B 2 1 ; MD LROI y end LB - { - 0x2018, - 0x9B, - }, - // - // // AE - // W 24 2100 01 2 1 ; [0]: AE control enable - { - 0x2100, - 0x01, - }, - // W 24 2101 07 2 1 ; AE target mean - { - 0x2101, - 0x5F, - }, - // W 24 2102 0A 2 1 ; AE min mean - { - 0x2102, - 0x0A, - }, - // W 24 2104 03 2 1 ; AE Threshold - { - 0x2103, - 0x03, - }, - // W 24 2104 05 2 1 ; AE Threshold - { - 0x2104, - 0x05, - }, - // W 24 2105 01 2 1 ; max INTG Hb - { - 0x2105, - 0x02, - }, - // W 24 2106 54 2 1 ; max INTG Lb - { - 0x2106, - 0x14, - }, - // W 24 2108 02 2 1 ; max AGain in full - { - 0x2107, - 0x02, - }, - // W 24 2108 03 2 1 ; max AGain in full - { - 0x2108, - 0x03, - }, - // W 24 2109 04 2 1 ; max AGain in bin2 - { - 0x2109, - 0x03, - }, - // W 24 210A 00 2 1 ; min AGAIN - { - 0x210A, - 0x00, - }, - // W 24 210B C0 2 1 ; max DGain - { - 0x210B, - 0x80, - }, - // W 24 210C 40 2 1 ; min DGain - { - 0x210C, - 0x40, - }, - // W 24 210D 20 2 1 ; damping factor - { - 0x210D, - 0x20, - }, - // W 24 210E 03 2 1 ; FS ctrl - { - 0x210E, - 0x03, - }, - // W 24 210F 00 2 1 ; FS 60Hz Hb - { - 0x210F, - 0x00, - }, - // W 24 2110 85 2 1 ; FS 60Hz Lb - { - 0x2110, - 0x85, - }, - // W 24 2111 00 2 1 ; Fs 50Hz Hb - { - 0x2111, - 0x00, - }, - // W 24 2112 A0 2 1 ; FS 50Hz Lb - { - 0x2112, - 0xA0, - }, - - // - // - // // MD - // W 24 2150 03 2 1 ; [0] : MD LROI en [1] : MD GROI en - { - 0x2150, - 0x03, - }, - // - // - // //--------------------------------------------------- - // // frame rate : 5 FPS - // //--------------------------------------------------- - // W 24 0340 0C 2 1 ; smia frame length Hb - { - 0x0340, - 0x0C, - }, - // W 24 0341 7A 2 1 ; smia frame length Lb 3192 - { - 0x0341, - 0x7A, - }, - // - // W 24 0342 01 2 1 ; smia line length Hb - { - 0x0342, - 0x01, - }, - // W 24 0343 77 2 1 ; smia line length Lb 375 - { - 0x0343, - 0x77, - }, - // - // //--------------------------------------------------- - // // Resolution : QVGA 324x244 - // //--------------------------------------------------- - // W 24 3010 01 2 1 ; [0] : window mode 0 : full frame 324x324 1 : QVGA - { - 0x3010, - 0x01, - }, - // - // - // W 24 0383 01 2 1 ; - { - 0x0383, - 0x01, - }, - // W 24 0387 01 2 1 ; - { - 0x0387, - 0x01, - }, - // W 24 0390 00 2 1 ; - { - 0x0390, - 0x00, - }, - // - // //--------------------------------------------------- - // // bit width Selection - // //--------------------------------------------------- - // W 24 3011 70 2 1 ; [0] : 6 bit mode enable - { - 0x3011, - 0x70, - }, - // - // - // W 24 3059 02 2 1 ; [7]: Self OSC En, [6]: 4bit mode, [5]: serial mode, - // [4:0]: keep value as 0x02 - { - 0x3059, - 0x02, - }, - // W 24 3060 01 2 1 ; [5]: gated_clock, [4]: msb first, - { - 0x3060, - 0x20, - }, - // ; [3:2]: vt_reg_div -> div by 4/8/1/2 - // ; [1;0]: vt_sys_div -> div by 8/4/2/1 - // - // - { - 0x0101, - 0x01, - }, - // //--------------------------------------------------- - // // CMU update - // //--------------------------------------------------- - // - // W 24 0104 01 2 1 ; was 0100 - { - 0x0104, - 0x01, - }, - // - // - // - // //--------------------------------------------------- - // // Turn on rolling shutter - // //--------------------------------------------------- - // W 24 0100 01 2 1 ; was 0005 ; mode_select 00 : standby - wait fir I2C SW - // trigger 01 : streaming 03 : output "N" frame, then enter standby 04 : - // standby - wait for HW trigger (level), then continuous video out til HW - // TRIG goes off 06 : standby - wait for HW trigger (edge), then output "N" - // frames then enter standby - { - 0x0100, - 0x01, - }, - // - // ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -}; - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_RAW8_QVGA_8BITS_LSB_5FPS_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_Walking1s_01.h b/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_Walking1s_01.h deleted file mode 100644 index 8818e249c17..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_Walking1s_01.h +++ /dev/null @@ -1,56 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_WALKING1S_01_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_WALKING1S_01_H_ - -#include "HM01B0.h" - -const hm_script_t sHM01b0TestModeScript_Walking1s[] = { - { - 0x2100, - 0x00, - }, // W 24 2100 00 2 1 ; AE - { - 0x1000, - 0x00, - }, // W 24 1000 00 2 1 ; BLC - { - 0x1008, - 0x00, - }, // W 24 1008 00 2 1 ; DPC - { - 0x0205, - 0x00, - }, // W 24 0205 00 2 1 ; AGain - { - 0x020E, - 0x01, - }, // W 24 020E 01 2 1 ; DGain - { - 0x020F, - 0x00, - }, // W 24 020F 00 2 1 ; DGain - { - 0x0601, - 0x11, - }, // W 24 0601 11 2 1 ; Test pattern - { - 0x0104, - 0x01, - }, // W 24 0104 01 2 1 ; -}; - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_WALKING1S_01_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_Walking1s_01.txt b/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_Walking1s_01.txt deleted file mode 100644 index 1244caddcac..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_Walking1s_01.txt +++ /dev/null @@ -1,8 +0,0 @@ -W 24 2100 00 2 1 ; AE -W 24 1000 00 2 1 ; BLC -W 24 1008 00 2 1 ; DPC -W 24 0205 00 2 1 ; AGain -W 24 020E 01 2 1 ; DGain -W 24 020F 00 2 1 ; DGain -W 24 0601 11 2 1 ; Test pattern -W 24 0104 01 2 1 ; diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.c b/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.c deleted file mode 100644 index bf897850ec3..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.c +++ /dev/null @@ -1,35 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "HM01B0_debug.h" -#include "am_util.h" // NOLINT - -void hm01b0_framebuffer_dump(uint8_t* frame, uint32_t length) { - am_util_stdio_printf("+++ frame +++"); - - for (uint32_t i = 0; i < length; i++) { - if ((i & 0xF) == 0x00) { - am_util_stdio_printf("\n0x%08LX ", i); - // this delay is to let itm have time to flush out data. - am_util_delay_ms(1); - } - - am_util_stdio_printf("%02X ", frame[i]); - } - - am_util_stdio_printf("\n--- frame ---\n"); - am_util_delay_ms(1); -} - diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.h b/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.h deleted file mode 100644 index 88d9a0a429e..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.h +++ /dev/null @@ -1,49 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_DEBUG_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_DEBUG_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "HM01B0.h" - -//***************************************************************************** -// -//! @brief Read one frame of data from HM01B0 scaled to 96x96 RGB. -//! -//! @param buffer - Pointer to the frame buffer. -//! @param w - Image width. -//! @param h - Image height. -//! @param channels - Number of channels per pixel. -//! -//! This function reads data of one frame from HM01B0. It trims the image to an -//! even power of two multiple of the requested width and height. It down -//! samples the original image and duplicates the greyscale value for each color -//! channel. -//! -//! @return Error code. -// -//***************************************************************************** - -void hm01b0_framebuffer_dump(uint8_t* frame, uint32_t len); - -#ifdef __cplusplus -} -#endif - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_HIMAX_DRIVER_HM01B0_DEBUG_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_optimized.c b/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_optimized.c deleted file mode 100644 index 3629c72b497..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_optimized.c +++ /dev/null @@ -1,87 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "HM01B0.h" -#include "am_bsp.h" //NOLINT -#include "am_mcu_apollo.h" //NOLINT -#include "platform_Sparkfun_Edge.h" - -// Image is down-sampled by applying a stride of 2 pixels in both the x and y -// directions. -static const int kStrideShift = 1; - -//***************************************************************************** -// -//! @brief Read one frame of data from HM01B0 scaled to 96x96 RGB. -//! -//! @param buffer - Pointer to the frame buffer. -//! @param w - Image width. -//! @param h - Image height. -//! @param channels - Number of channels per pixel. -//! -//! This function reads data of one frame from HM01B0. It trims the image to an -//! even power of two mulitple of the requested width and height. It down -//! samples the original image and duplicates the greyscale value for each color -//! channel. -//! -//! @return Error code. -// -//***************************************************************************** -uint32_t hm01b0_blocking_read_oneframe_scaled(hm01b0_cfg_t* psCfg, - int8_t* buffer, int w, int h, - int channels) { - hm01b0_single_frame_capture(psCfg); - - // Calculate the number of pixels to crop to get a centered image. - const int offset_x = (HM01B0_PIXEL_X_NUM - (w * (1 << kStrideShift))) / 2; - const int offset_y = (HM01B0_PIXEL_Y_NUM - (h * (1 << kStrideShift))) / 2; - - uint32_t hsync_count = 0; - - while ((hsync_count < HM01B0_PIXEL_Y_NUM)) { - // Wait for horizontal sync. - while (!read_hsync()); - - // Get resulting image position. When hsync_count < offset_y, this will - // underflow resulting in an index out of bounds which we check later, - // avoiding an unnecessary conditional. - const uint32_t output_y = (hsync_count - offset_y) >> kStrideShift; - uint32_t rowidx = 0; - - // Read one row. Hsync is held high for the duration of a row read. - while (read_hsync()) { - // Wait for pixel value to be ready. - while (!read_pclk()); - - // Read 8-bit value from camera. - const uint8_t value = read_byte(); - const uint32_t output_x = (rowidx++ - offset_x) >> kStrideShift; - if (output_x < w && output_y < h) { - const int output_idx = (output_y * w + output_x) * channels; - for (int i=0; i - -#include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h" -#include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/testing/micro_test.h" - -TF_LITE_MICRO_TESTS_BEGIN - -TF_LITE_MICRO_TEST(TestImageProvider) { - tflite::MicroErrorReporter micro_error_reporter; - - int8_t image_data[kMaxImageSize]; - TfLiteStatus get_status = GetImage(µ_error_reporter, kNumCols, kNumRows, - kNumChannels, image_data); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, get_status); - TF_LITE_MICRO_EXPECT_NE(image_data, nullptr); - - // Make sure we can read all of the returned memory locations. - uint32_t total = 0; - for (int i = 0; i < kMaxImageSize; ++i) { - total += image_data[i]; - } -} - -TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/main.cc b/tensorflow/lite/micro/examples/person_detection_experimental/main.cc deleted file mode 100644 index 603a3a288f8..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/main.cc +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/main_functions.h" - -// This is the default main used on systems that have the standard C entry -// point. Other devices (for example FreeRTOS or ESP32) that have different -// requirements for entry code (like an app_main function) should specialize -// this main.cc file in a target-specific subfolder. -int main(int argc, char* argv[]) { - setup(); - while (true) { - loop(); - } -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc b/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc deleted file mode 100644 index f1ded80d1b9..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.cc +++ /dev/null @@ -1,117 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/main_functions.h" - -#include "tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/image_provider.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h" -#include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/micro_interpreter.h" -#include "tensorflow/lite/micro/micro_mutable_op_resolver.h" -#include "tensorflow/lite/schema/schema_generated.h" -#include "tensorflow/lite/version.h" - -// Globals, used for compatibility with Arduino-style sketches. -namespace { -tflite::ErrorReporter* error_reporter = nullptr; -const tflite::Model* model = nullptr; -tflite::MicroInterpreter* interpreter = nullptr; -TfLiteTensor* input = nullptr; - -// In order to use optimized tensorflow lite kernels, a signed int8_t quantized -// model is preferred over the legacy unsigned model format. This means that -// throughout this project, input images must be converted from unisgned to -// signed format. The easiest and quickest way to convert from unsigned to -// signed 8-bit integers is to subtract 128 from the unsigned value to get a -// signed value. - -// An area of memory to use for input, output, and intermediate arrays. -constexpr int kTensorArenaSize = 136 * 1024; -static uint8_t tensor_arena[kTensorArenaSize]; -} // namespace - -// The name of this function is important for Arduino compatibility. -void setup() { - // Set up logging. Google style is to avoid globals or statics because of - // lifetime uncertainty, but since this has a trivial destructor it's okay. - // NOLINTNEXTLINE(runtime-global-variables) - static tflite::MicroErrorReporter micro_error_reporter; - error_reporter = µ_error_reporter; - - // Map the model into a usable data structure. This doesn't involve any - // copying or parsing, it's a very lightweight operation. - model = tflite::GetModel(g_person_detect_model_data); - if (model->version() != TFLITE_SCHEMA_VERSION) { - TF_LITE_REPORT_ERROR(error_reporter, - "Model provided is schema version %d not equal " - "to supported version %d.", - model->version(), TFLITE_SCHEMA_VERSION); - return; - } - - // Pull in only the operation implementations we need. - // This relies on a complete list of all the ops needed by this graph. - // An easier approach is to just use the AllOpsResolver, but this will - // incur some penalty in code space for op implementations that are not - // needed by this graph. - // - // tflite::AllOpsResolver resolver; - // NOLINTNEXTLINE(runtime-global-variables) - static tflite::MicroMutableOpResolver<5> micro_op_resolver; - micro_op_resolver.AddAveragePool2D(); - micro_op_resolver.AddConv2D(); - micro_op_resolver.AddDepthwiseConv2D(); - micro_op_resolver.AddReshape(); - micro_op_resolver.AddSoftmax(); - - // Build an interpreter to run the model with. - // NOLINTNEXTLINE(runtime-global-variables) - static tflite::MicroInterpreter static_interpreter( - model, micro_op_resolver, tensor_arena, kTensorArenaSize, error_reporter); - interpreter = &static_interpreter; - - // Allocate memory from the tensor_arena for the model's tensors. - TfLiteStatus allocate_status = interpreter->AllocateTensors(); - if (allocate_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(error_reporter, "AllocateTensors() failed"); - return; - } - - // Get information about the memory area to use for the model's input. - input = interpreter->input(0); -} - -// The name of this function is important for Arduino compatibility. -void loop() { - // Get image from provider. - if (kTfLiteOk != GetImage(error_reporter, kNumCols, kNumRows, kNumChannels, - input->data.int8)) { - TF_LITE_REPORT_ERROR(error_reporter, "Image capture failed."); - } - - // Run the model on this input and make sure it succeeds. - if (kTfLiteOk != interpreter->Invoke()) { - TF_LITE_REPORT_ERROR(error_reporter, "Invoke failed."); - } - - TfLiteTensor* output = interpreter->output(0); - - // Process the inference results. - int8_t person_score = output->data.uint8[kPersonIndex]; - int8_t no_person_score = output->data.uint8[kNotAPersonIndex]; - RespondToDetection(error_reporter, person_score, no_person_score); -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.h b/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.h deleted file mode 100644 index 7bfedf18524..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/main_functions.h +++ /dev/null @@ -1,28 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_MAIN_FUNCTIONS_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_MAIN_FUNCTIONS_H_ - -// Initializes all data needed for the example. The name is important, and needs -// to be setup() for Arduino compatibility. -void setup(); - -// Runs one iteration of data gathering and inference. This should be called -// repeatedly from the application code. The name needs to be loop() for Arduino -// compatibility. -void loop(); - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_MAIN_FUNCTIONS_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/model_settings.cc b/tensorflow/lite/micro/examples/person_detection_experimental/model_settings.cc deleted file mode 100644 index c7359b8fb5d..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/model_settings.cc +++ /dev/null @@ -1,21 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h" - -const char* kCategoryLabels[kCategoryCount] = { - "notperson", - "person", -}; diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h b/tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h deleted file mode 100644 index f6c968e99b6..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h +++ /dev/null @@ -1,35 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_MODEL_SETTINGS_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_MODEL_SETTINGS_H_ - -// Keeping these as constant expressions allow us to allocate fixed-sized arrays -// on the stack for our working memory. - -// All of these values are derived from the values used during model training, -// if you change your model you'll need to update these constants. -constexpr int kNumCols = 96; -constexpr int kNumRows = 96; -constexpr int kNumChannels = 1; - -constexpr int kMaxImageSize = kNumCols * kNumRows * kNumChannels; - -constexpr int kCategoryCount = 2; -constexpr int kPersonIndex = 1; -constexpr int kNotAPersonIndex = 0; -extern const char* kCategoryLabels[kCategoryCount]; - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_MODEL_SETTINGS_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/no_person_image_data.h b/tensorflow/lite/micro/examples/person_detection_experimental/no_person_image_data.h deleted file mode 100644 index d3db7beb210..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/no_person_image_data.h +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// This data was created from a sample image from without a person in it. -// Convert original image to simpler format: -// convert -resize 96x96\! noperson.PNG noperson.bmp3 -// Skip the 54 byte bmp3 header and add the reset of the bytes to a C array: -// xxd -s 54 -i /tmp/noperson.bmp3 > /tmp/noperson.cc - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_NO_PERSON_IMAGE_DATA_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_NO_PERSON_IMAGE_DATA_H_ - -#include - -extern const int g_no_person_data_size; -extern const uint8_t g_no_person_data[]; - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_NO_PERSON_IMAGE_DATA_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h b/tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h deleted file mode 100644 index 5d1b59ffdc9..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// This is a standard TensorFlow Lite model file that has been converted into a -// C data array, so it can be easily compiled into a binary for devices that -// don't have a file system. It was created using the command: -// xxd -i person_detect.tflite > person_detect_model_data.cc - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_PERSON_DETECT_MODEL_DATA_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_PERSON_DETECT_MODEL_DATA_H_ - -extern const unsigned char g_person_detect_model_data[]; -extern const int g_person_detect_model_data_len; - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_PERSON_DETECT_MODEL_DATA_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc b/tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc deleted file mode 100644 index 6175a59ba52..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/person_detection_test.cc +++ /dev/null @@ -1,135 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/model_settings.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/no_person_image_data.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/person_detect_model_data.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/person_image_data.h" -#include "tensorflow/lite/micro/micro_error_reporter.h" -#include "tensorflow/lite/micro/micro_interpreter.h" -#include "tensorflow/lite/micro/micro_mutable_op_resolver.h" -#include "tensorflow/lite/micro/testing/micro_test.h" -#include "tensorflow/lite/schema/schema_generated.h" -#include "tensorflow/lite/version.h" - -// Create an area of memory to use for input, output, and intermediate arrays. -constexpr int tensor_arena_size = 136 * 1024; -uint8_t tensor_arena[tensor_arena_size]; - -TF_LITE_MICRO_TESTS_BEGIN - -TF_LITE_MICRO_TEST(TestInvoke) { - // Set up logging. - tflite::MicroErrorReporter micro_error_reporter; - - // Map the model into a usable data structure. This doesn't involve any - // copying or parsing, it's a very lightweight operation. - const tflite::Model* model = ::tflite::GetModel(g_person_detect_model_data); - if (model->version() != TFLITE_SCHEMA_VERSION) { - TF_LITE_REPORT_ERROR(µ_error_reporter, - "Model provided is schema version %d not equal " - "to supported version %d.\n", - model->version(), TFLITE_SCHEMA_VERSION); - } - - // Pull in only the operation implementations we need. - // This relies on a complete list of all the ops needed by this graph. - // An easier approach is to just use the AllOpsResolver, but this will - // incur some penalty in code space for op implementations that are not - // needed by this graph. - tflite::MicroMutableOpResolver<5> micro_op_resolver; - micro_op_resolver.AddAveragePool2D(); - micro_op_resolver.AddConv2D(); - micro_op_resolver.AddDepthwiseConv2D(); - micro_op_resolver.AddReshape(); - micro_op_resolver.AddSoftmax(); - - // Build an interpreter to run the model with. - tflite::MicroInterpreter interpreter(model, micro_op_resolver, tensor_arena, - tensor_arena_size, - µ_error_reporter); - interpreter.AllocateTensors(); - - // Get information about the memory area to use for the model's input. - TfLiteTensor* input = interpreter.input(0); - - // Make sure the input has the properties we expect. - TF_LITE_MICRO_EXPECT_NE(nullptr, input); - TF_LITE_MICRO_EXPECT_EQ(4, input->dims->size); - TF_LITE_MICRO_EXPECT_EQ(1, input->dims->data[0]); - TF_LITE_MICRO_EXPECT_EQ(kNumRows, input->dims->data[1]); - TF_LITE_MICRO_EXPECT_EQ(kNumCols, input->dims->data[2]); - TF_LITE_MICRO_EXPECT_EQ(kNumChannels, input->dims->data[3]); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteInt8, input->type); - - // Copy an image with a person into the memory area used for the input. - TFLITE_DCHECK_EQ(input->bytes, static_cast(g_person_data_size)); - memcpy(input->data.int8, g_person_data, input->bytes); - - // Run the model on this input and make sure it succeeds. - TfLiteStatus invoke_status = interpreter.Invoke(); - if (invoke_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(µ_error_reporter, "Invoke failed\n"); - } - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, invoke_status); - - // Get the output from the model, and make sure it's the expected size and - // type. - TfLiteTensor* output = interpreter.output(0); - TF_LITE_MICRO_EXPECT_EQ(2, output->dims->size); - TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[0]); - TF_LITE_MICRO_EXPECT_EQ(kCategoryCount, output->dims->data[1]); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteInt8, output->type); - - // Make sure that the expected "Person" score is higher than the other class. - int8_t person_score = output->data.int8[kPersonIndex]; - int8_t no_person_score = output->data.int8[kNotAPersonIndex]; - TF_LITE_REPORT_ERROR(µ_error_reporter, - "person data. person score: %d, no person score: %d\n", - person_score, no_person_score); - TF_LITE_MICRO_EXPECT_GT(person_score, no_person_score); - - // TODO(b/161461076): Update model to make this work on real negative inputs. - memset(input->data.int8, 0, input->bytes); - - // Run the model on this "No Person" input. - invoke_status = interpreter.Invoke(); - if (invoke_status != kTfLiteOk) { - TF_LITE_REPORT_ERROR(µ_error_reporter, "Invoke failed\n"); - } - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, invoke_status); - - // Get the output from the model, and make sure it's the expected size and - // type. - output = interpreter.output(0); - TF_LITE_MICRO_EXPECT_EQ(2, output->dims->size); - TF_LITE_MICRO_EXPECT_EQ(1, output->dims->data[0]); - TF_LITE_MICRO_EXPECT_EQ(kCategoryCount, output->dims->data[1]); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteInt8, output->type); - - // Make sure that the expected "No Person" score is higher. - person_score = output->data.int8[kPersonIndex]; - no_person_score = output->data.int8[kNotAPersonIndex]; - TF_LITE_REPORT_ERROR( - µ_error_reporter, - "no person data. person score: %d, no person score: %d\n", person_score, - no_person_score); - TF_LITE_MICRO_EXPECT_GT(no_person_score, person_score); - - TF_LITE_REPORT_ERROR(µ_error_reporter, "Ran successfully\n"); -} - -TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/person_image_data.h b/tensorflow/lite/micro/examples/person_detection_experimental/person_image_data.h deleted file mode 100644 index 13e16666bc6..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/person_image_data.h +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// This data was created from a sample image from with a person in it. -// Convert original image to simpler format: -// convert -resize 96x96\! person.PNG person.bmp3 -// Skip the 54 byte bmp3 header and add the reset of the bytes to a C array: -// xxd -s 54 -i /tmp/person.bmp3 > /tmp/person.cc - -#ifndef TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_PERSON_IMAGE_DATA_H_ -#define TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_PERSON_IMAGE_DATA_H_ - -#include - -extern const int g_person_data_size; -extern const uint8_t g_person_data[]; - -#endif // TENSORFLOW_LITE_MICRO_EXAMPLES_PERSON_DETECTION_EXPERIMENTAL_PERSON_IMAGE_DATA_H_ diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/sparkfun_edge/detection_responder.cc b/tensorflow/lite/micro/examples/person_detection_experimental/sparkfun_edge/detection_responder.cc deleted file mode 100644 index 7a85c0132c8..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/sparkfun_edge/detection_responder.cc +++ /dev/null @@ -1,54 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/detection_responder.h" - -#include "am_bsp.h" // NOLINT - -// This implementation will light up LEDs on the board in response to the -// inference results. -void RespondToDetection(tflite::ErrorReporter* error_reporter, - int8_t person_score, int8_t no_person_score) { - static bool is_initialized = false; - if (!is_initialized) { - // Setup LED's as outputs. Leave red LED alone since that's an error - // indicator for sparkfun_edge in image_provider. - am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_BLUE, g_AM_HAL_GPIO_OUTPUT_12); - am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_GREEN, g_AM_HAL_GPIO_OUTPUT_12); - am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_YELLOW, g_AM_HAL_GPIO_OUTPUT_12); - is_initialized = true; - } - - // Toggle the blue LED every time an inference is performed. - static int count = 0; - if (++count & 1) { - am_hal_gpio_output_set(AM_BSP_GPIO_LED_BLUE); - } else { - am_hal_gpio_output_clear(AM_BSP_GPIO_LED_BLUE); - } - - // Turn on the green LED if a person was detected. Turn on the yellow LED - // otherwise. - am_hal_gpio_output_clear(AM_BSP_GPIO_LED_YELLOW); - am_hal_gpio_output_clear(AM_BSP_GPIO_LED_GREEN); - if (person_score > no_person_score) { - am_hal_gpio_output_set(AM_BSP_GPIO_LED_GREEN); - } else { - am_hal_gpio_output_set(AM_BSP_GPIO_LED_YELLOW); - } - - TF_LITE_REPORT_ERROR(error_reporter, "Person score: %d No person score: %d", - person_score, no_person_score); -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/sparkfun_edge/image_provider.cc b/tensorflow/lite/micro/examples/person_detection_experimental/sparkfun_edge/image_provider.cc deleted file mode 100644 index 01f39b41d12..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/sparkfun_edge/image_provider.cc +++ /dev/null @@ -1,203 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/lite/micro/examples/person_detection_experimental/image_provider.h" - -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_RAW8_QVGA_8bits_lsb_5fps.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_debug.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/HM01B0_optimized.h" -#include "tensorflow/lite/micro/examples/person_detection_experimental/himax_driver/platform_Sparkfun_Edge.h" - -// These are headers from Ambiq's Apollo3 SDK. -#include "am_bsp.h" // NOLINT -#include "am_mcu_apollo.h" // NOLINT -#include "am_util.h" // NOLINT - -// #define DEMO_HM01B0_FRAMEBUFFER_DUMP_ENABLE - -// Enabling logging increases power consumption by preventing low power mode -// from being enabled. -#define ENABLE_LOGGING - -namespace { - -//***************************************************************************** -// -// HM01B0 Configuration -// -//***************************************************************************** -static hm01b0_cfg_t s_HM01B0Cfg = { - // i2c settings - ui16SlvAddr : HM01B0_DEFAULT_ADDRESS, - eIOMMode : HM01B0_IOM_MODE, - ui32IOMModule : HM01B0_IOM_MODULE, - sIOMCfg : { - eInterfaceMode : HM01B0_IOM_MODE, - ui32ClockFreq : HM01B0_I2C_CLOCK_FREQ, - }, - pIOMHandle : NULL, - - // MCLK settings - ui32CTimerModule : HM01B0_MCLK_GENERATOR_MOD, - ui32CTimerSegment : HM01B0_MCLK_GENERATOR_SEG, - ui32CTimerOutputPin : HM01B0_PIN_MCLK, - - // data interface - ui8PinSCL : HM01B0_PIN_SCL, - ui8PinSDA : HM01B0_PIN_SDA, - ui8PinD0 : HM01B0_PIN_D0, - ui8PinD1 : HM01B0_PIN_D1, - ui8PinD2 : HM01B0_PIN_D2, - ui8PinD3 : HM01B0_PIN_D3, - ui8PinD4 : HM01B0_PIN_D4, - ui8PinD5 : HM01B0_PIN_D5, - ui8PinD6 : HM01B0_PIN_D6, - ui8PinD7 : HM01B0_PIN_D7, - ui8PinVSYNC : HM01B0_PIN_VSYNC, - ui8PinHSYNC : HM01B0_PIN_HSYNC, - ui8PinPCLK : HM01B0_PIN_PCLK, - - ui8PinTrig : HM01B0_PIN_TRIG, - ui8PinInt : HM01B0_PIN_INT, - pfnGpioIsr : NULL, -}; - -static constexpr int kFramesToInitialize = 4; - -bool g_is_camera_initialized = false; - -void burst_mode_enable(tflite::ErrorReporter* error_reporter, bool bEnable) { - am_hal_burst_avail_e eBurstModeAvailable; - am_hal_burst_mode_e eBurstMode; - - // Check that the Burst Feature is available. - if (AM_HAL_STATUS_SUCCESS == - am_hal_burst_mode_initialize(&eBurstModeAvailable)) { - if (AM_HAL_BURST_AVAIL == eBurstModeAvailable) { - TF_LITE_REPORT_ERROR(error_reporter, "Apollo3 Burst Mode is Available\n"); - } else { - TF_LITE_REPORT_ERROR(error_reporter, - "Apollo3 Burst Mode is Not Available\n"); - return; - } - } else { - TF_LITE_REPORT_ERROR(error_reporter, - "Failed to Initialize for Burst Mode operation\n"); - } - - // Make sure we are in "Normal" mode. - if (AM_HAL_STATUS_SUCCESS == am_hal_burst_mode_disable(&eBurstMode)) { - if (AM_HAL_NORMAL_MODE == eBurstMode) { - TF_LITE_REPORT_ERROR(error_reporter, - "Apollo3 operating in Normal Mode (48MHz)\n"); - } - } else { - TF_LITE_REPORT_ERROR(error_reporter, - "Failed to Disable Burst Mode operation\n"); - } - - // Put the MCU into "Burst" mode. - if (bEnable) { - if (AM_HAL_STATUS_SUCCESS == am_hal_burst_mode_enable(&eBurstMode)) { - if (AM_HAL_BURST_MODE == eBurstMode) { - TF_LITE_REPORT_ERROR(error_reporter, - "Apollo3 operating in Burst Mode (96MHz)\n"); - } - } else { - TF_LITE_REPORT_ERROR(error_reporter, - "Failed to Enable Burst Mode operation\n"); - } - } -} - -} // namespace - -TfLiteStatus InitCamera(tflite::ErrorReporter* error_reporter) { - TF_LITE_REPORT_ERROR(error_reporter, "Initializing HM01B0...\n"); - - am_hal_clkgen_control(AM_HAL_CLKGEN_CONTROL_SYSCLK_MAX, 0); - - // Set the default cache configuration - am_hal_cachectrl_config(&am_hal_cachectrl_defaults); - am_hal_cachectrl_enable(); - - // Configure the board for low power operation. This breaks logging by - // turning off the itm and uart interfaces. -#ifndef ENABLE_LOGGING - am_bsp_low_power_init(); -#endif - - // Enable interrupts so we can receive messages from the boot host. - am_hal_interrupt_master_enable(); - - burst_mode_enable(error_reporter, true); - - // Turn on the 1.8V regulator for DVDD on the camera. - am_hal_gpio_pinconfig(HM01B0_PIN_DVDD_EN, g_AM_HAL_GPIO_OUTPUT_12); - am_hal_gpio_output_set(HM01B0_PIN_DVDD_EN); - - // Configure Red LED for debugging. - am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_RED, g_AM_HAL_GPIO_OUTPUT_12); - am_hal_gpio_output_clear(AM_BSP_GPIO_LED_RED); - - hm01b0_power_up(&s_HM01B0Cfg); - - am_util_delay_ms(1); - - hm01b0_mclk_enable(&s_HM01B0Cfg); - - am_util_delay_ms(1); - - if (HM01B0_ERR_OK != hm01b0_init_if(&s_HM01B0Cfg)) { - return kTfLiteError; - } - - if (HM01B0_ERR_OK != - hm01b0_init_system(&s_HM01B0Cfg, (hm_script_t*)sHM01B0InitScript, - sizeof(sHM01B0InitScript) / sizeof(hm_script_t))) { - return kTfLiteError; - } - - return kTfLiteOk; -} - -// Capture single frame. Frame pointer passed in to reduce memory usage. This -// allows the input tensor to be used instead of requiring an extra copy. -TfLiteStatus GetImage(tflite::ErrorReporter* error_reporter, int frame_width, - int frame_height, int channels, int8_t* frame) { - if (!g_is_camera_initialized) { - TfLiteStatus init_status = InitCamera(error_reporter); - if (init_status != kTfLiteOk) { - am_hal_gpio_output_set(AM_BSP_GPIO_LED_RED); - return init_status; - } - // Drop a few frames until auto exposure is calibrated. - for (int i = 0; i < kFramesToInitialize; ++i) { - hm01b0_blocking_read_oneframe_scaled(&s_HM01B0Cfg, frame, frame_width, - frame_height, channels); - } - g_is_camera_initialized = true; - } - - hm01b0_blocking_read_oneframe_scaled(&s_HM01B0Cfg, frame, frame_width, - frame_height, channels); - -#ifdef DEMO_HM01B0_FRAMEBUFFER_DUMP_ENABLE - hm01b0_framebuffer_dump(frame, frame_width * frame_height * channels); -#endif - - return kTfLiteOk; -} diff --git a/tensorflow/lite/micro/examples/person_detection_experimental/training_a_model.md b/tensorflow/lite/micro/examples/person_detection_experimental/training_a_model.md deleted file mode 100644 index 81ac39bbf14..00000000000 --- a/tensorflow/lite/micro/examples/person_detection_experimental/training_a_model.md +++ /dev/null @@ -1,455 +0,0 @@ -## Training a model - -The following document will walk you through the process of training your own -250 KB embedded vision model using scripts that are easy to run. You can use -either the [Visual Wake Words dataset](https://arxiv.org/abs/1906.05721) for -person detection, or choose one of the [80 -categories from the MSCOCO dataset](http://cocodataset.org/#explore). - -This model will take several days to train on a powerful machine with GPUs. We -recommend using a [Google Cloud Deep -Learning VM](https://cloud.google.com/deep-learning-vm/). - -### Training framework choice - -Keras is the recommended interface for building models in TensorFlow, but when -the person detector model was being created it didn't yet support all the -features we needed. For that reason, we'll be showing you how to train a model -using tf.slim, an older interface. It is still widely used but deprecated, so -future versions of TensorFlow may not support this approach. We hope to publish -Keras instructions in the future. - -The model definitions for Slim are part of the -[TensorFlow models repository](https://github.com/tensorflow/models), so to get -started you'll need to download it from GitHub using a command like this: - -``` -! cd ~ -! git clone https://github.com/tensorflow/models.git -``` - -The following guide is going to assume that you've done this from your home -directory, so the model repository code is at ~/models, and that all commands -are run from the home directory too unless otherwise noted. You can place the -repository somewhere else, but you'll need to update all references to it. - -To use Slim, you'll need to make sure its modules can be found by Python, and -install one dependency. Here's how to do this in an iPython notebook: - -``` -! pip install contextlib2 -import os -new_python_path = (os.environ.get("PYTHONPATH") or '') + ":models/research/slim" -%env PYTHONPATH=$new_python_path -``` - -Updating `PYTHONPATH` through an `EXPORT` statement like this only works for the -current Jupyter session, so if you're using bash directly, you should add it to -a persistent startup script, running something like this: - -``` -echo 'export PYTHONPATH=$PYTHONPATH:models/research/slim' >> ~/.bashrc -source ~/.bashrc -``` - -If you see import errors running the slim scripts, you should make sure the -`PYTHONPATH` is set up correctly, and that contextlib2 has been installed. You -can find more general information on tf.slim in the -[repository's -README](https://github.com/tensorflow/models/tree/master/research/slim). - -### Building the dataset - -In order to train a person detector model, we need a large collection of images -that are labeled depending on whether or not they have people in them. The -ImageNet one-thousand class data that's widely used for training image -classifiers doesn't include labels for people, but luckily the -[COCO dataset](http://cocodataset.org/#home) does. You can also download this -data without manually registering too, and Slim provides a convenient script to -grab it automatically: - -``` -! chmod +x models/research/slim/datasets/download_mscoco.sh -! bash models/research/slim/datasets/download_mscoco.sh coco -``` - -This is a large download, about 40GB, so it will take a while and you'll need -to make sure you have at least 100GB free on your drive to allow space for -unpacking and further processing. The argument to the script is the path that -the data will be downloaded to. If you change this, you'll also need to update -the commands below that use it. - -The dataset is designed to be used for training models for localization, so the -images aren't labeled with the "contains a person", "doesn't contain a person" -categories that we want to train for. Instead each image comes with a list of -bounding boxes for all of the objects it contains. "Person" is one of these -object categories, so to get to the classification labels we want, we have to -look for images with bounding boxes for people. To make sure that they aren't -too tiny to be recognizable we also need to exclude very small bounding boxes. -Slim contains a script to convert the bounding box into labels: - -``` -! python models/research/slim/datasets/build_visualwakewords_data.py ---logtostderr \ ---train_image_dir=coco/raw-data/train2014 \ ---val_image_dir=coco/raw-data/val2014 \ ---train_annotations_file=coco/raw-data/annotations/instances_train2014.json \ ---val_annotations_file=coco/raw-data/annotations/instances_val2014.json \ ---output_dir=coco/processed \ ---small_object_area_threshold=0.005 \ ---foreground_class_of_interest='person' -``` - -Don't be surprised if this takes up to twenty minutes to complete. When it's -done, you'll have a set of TFRecords in `coco/processed` holding the labeled -image information. This data was created by Aakanksha Chowdhery and is known as -the [Visual Wake Words dataset](https://arxiv.org/abs/1906.05721). It's designed -to be useful for benchmarking and testing embedded computer vision, since it -represents a very common task that we need to accomplish with tight resource -constraints. We're hoping to see it drive even better models for this and -similar tasks. - -### Training the model - -One of the nice things about using tf.slim to handle the training is that the -parameters you commonly need to modify are available as command line arguments, -so we can just call the standard `train_image_classifier.py` script to train -our model. You can use this command to build the model we use in the example: - -``` -! python models/research/slim/train_image_classifier.py \ - --train_dir=vww_96_grayscale \ - --dataset_name=visualwakewords \ - --dataset_split_name=train \ - --dataset_dir=coco/processed \ - --model_name=mobilenet_v1_025 \ - --preprocessing_name=mobilenet_v1 \ - --train_image_size=96 \ - --input_grayscale=True \ - --save_summaries_secs=300 \ - --learning_rate=0.045 \ - --label_smoothing=0.1 \ - --learning_rate_decay_factor=0.98 \ - --num_epochs_per_decay=2.5 \ - --moving_average_decay=0.9999 \ - --batch_size=96 \ - --max_number_of_steps=1000000 -``` - -This will take a couple of days on a single-GPU v100 instance to complete all -one-million steps, but you should be able to get a fairly accurate model after -a few hours if you want to experiment early. - -- The checkpoints and summaries will the saved in the folder given in the - `--train_dir` argument, so that's where you'll have to look for the results. -- The `--dataset_dir` parameter should match the one where you saved the - TFRecords from the Visual Wake Words build script. -- The architecture we'll be using is defined by the `--model_name` argument. - The 'mobilenet_v1' prefix tells the script to use the first version of - MobileNet. We did experiment with later versions, but these used more RAM - for their intermediate activation buffers, so for now we kept with the - original. The '025' is the depth multiplier to use, which mostly affects the - number of weight parameters, this low setting ensures the model fits within - 250KB of Flash. -- `--preprocessing_name` controls how input images are modified before they're - fed into the model. The 'mobilenet_v1' version shrinks the width and height - of the images to the size given in `--train_image_size` (in our case 96 - pixels since we want to reduce the compute requirements). It also scales the - pixel values from 0 to 255 integers into -1.0 to +1.0 floating point numbers - (though we'll be quantizing those after training). -- The - [HM01B0](https://himax.com.tw/products/cmos-image-sensor/image-sensors/hm01b0/) - camera we're using on the SparkFun Edge board is monochrome, so to get the - best results we have to train our model on black and white images too, so we - pass in the `--input_grayscale` flag to enable that preprocessing. -- The `--learning_rate`, `--label_smoothing`, `--learning_rate_decay_factor`, - `--num_epochs_per_decay`, `--moving_average_decay` and `--batch_size` are - all parameters that control how weights are updated during the training - process. Training deep networks is still a bit of a dark art, so these exact - values we found through experimentation for this particular model. You can - try tweaking them to speed up training or gain a small boost in accuracy, - but we can't give much guidance for how to make those changes, and it's easy - to get combinations where the training accuracy never converges. -- The `--max_number_of_steps` defines how long the training should continue. - There's no good way to figure out this threshold in advance, you have to - experiment to tell when the accuracy of the model is no longer improving to - tell when to cut it off. In our case we default to a million steps, since - with this particular model we know that's a good point to stop. - -Once you start the script, you should see output that looks something like this: - -``` -INFO:tensorflow:global step 4670: loss = 0.7112 (0.251 sec/step) -I0928 00:16:21.774756 140518023943616 learning.py:507] global step 4670: loss = -0.7112 (0.251 sec/step) -INFO:tensorflow:global step 4680: loss = 0.6596 (0.227 sec/step) -I0928 00:16:24.365901 140518023943616 learning.py:507] global step 4680: loss = -0.6596 (0.227 sec/step) -``` - -Don't worry about the line duplication, this is just a side-effect of the way -TensorFlow log printing interacts with Python. Each line has two key bits of -information about the training process. The global step is a count of how far -through the training we are. Since we've set the limit as a million steps, in -this case we're nearly five percent complete. The steps per second estimate is -also useful, since you can use it to estimate a rough duration for the whole -training process. In this case, we're completing about four steps a second, so -a million steps will take about 70 hours, or three days. The other crucial -piece of information is the loss. This is a measure of how close the -partially-trained model's predictions are to the correct values, and lower -values are better. This will show a lot of variation but should on average -decrease during training if the model is learning. Because it's so noisy, the -amounts will bounce around a lot over short time periods, but if things are -working well you should see a noticeable drop if you wait an hour or so and -check back. This kind of variation is a lot easier to see in a graph, which is -one of the main reasons to try TensorBoard. - -### TensorBoard - -TensorBoard is a web application that lets you view data visualizations from -TensorFlow training sessions, and it's included by default in most cloud -instances. If you're using Google Cloud's AI Platform, you can start up a new -TensorBoard session by open the command palette from the left tabs on the -notebook interface, and scrolling down to select "Create a new tensorboard". -You'll be prompted for the location of the summary logs, enter the path you -used for `--train_dir` in the training script, in our example -'vww_96_grayscale'. One common error to watch out for is adding a slash to the -end of the path, which will cause tensorboard to fail to find the directory. If -you're starting tensorboard from the command line in a different environment -you'll have to pass in this path as the `--logdir` argument to the tensorboard -command line tool, and point your browser to http://localhost:6006 (or the -address of the machine you're running it on). - -It may take a little while for the graphs to have anything useful in them, since -the script only saves summaries every five minutes. The most important graph is -called 'clone_loss', and this shows the progression of the same loss value -that's displayed on the logging output. It fluctuates a lot, but the -overall trend is downwards over time. If you don't see this sort of progression -after a few hours of training, it's a good sign that your model isn't -converging to a good solution, and you may need to debug what's going wrong -either with your dataset or the training parameters. - -Tensorboard defaults to the 'Scalars' tab when it opens, but the other section -that can be useful during training is 'Images'. This shows a -random selection of the pictures the model is currently being trained on, -including any distortions and other preprocessing. This information isn't as -essential as the loss graphs, but it can be useful to ensure the dataset is what -you expect, and it is interesting to see the examples updating as training -progresses. - -### Evaluating the model - -The loss function correlates with how well your model is training, but it isn't -a direct, understandable metric. What we really care about is how many people -our model detects correctly, but to get calculate this we need to run a -separate script. You don't need to wait until the model is fully trained, you -can check the accuracy of any checkpoints in the `--train_dir` folder. - -``` -! python models/research/slim/eval_image_classifier.py \ - --alsologtostderr \ - --checkpoint_path=vww_96_grayscale/model.ckpt-698580 \ - --dataset_dir=coco/processed/ \ - --dataset_name=visualwakewords \ - --dataset_split_name=val \ - --model_name=mobilenet_v1_025 \ - --preprocessing_name=mobilenet_v1 \ - --input_grayscale=True \ - --train_image_size=96 -``` - -You'll need to make sure that `--checkpoint_path` is pointing to a valid set of -checkpoint data. Checkpoints are stored in three separate files, so the value -should be their common prefix. For example if you have a checkpoint file called -'model.ckpt-5179.data-00000-of-00001', the prefix would be 'model.ckpt-5179'. -The script should produce output that looks something like this: - -``` -INFO:tensorflow:Evaluation [406/406] -I0929 22:52:59.936022 140225887045056 evaluation.py:167] Evaluation [406/406] -eval/Accuracy[0.717438412]eval/Recall_5[1] -``` - -The important number here is the accuracy. It shows the proportion of the -images that were classified correctly, which is 72% in this case, after -converting to a percentage. If you follow the example script, you should expect -a fully-trained model to achieve an accuracy of around 84% after one million -steps, and show a loss of around 0.4. - -### Exporting the model to TensorFlow Lite - -When the model has trained to an accuracy you're happy with, you'll need to -convert the results from the TensorFlow training environment into a form you -can run on an embedded device. As we've seen in previous chapters, this can be -a complex process, and tf.slim adds a few of its own wrinkles too. - -#### Exporting to a GraphDef protobuf file - -Slim generates the architecture from the model_name every time one of its -scripts is run, so for a model to be used outside of Slim it needs to be saved -in a common format. We're going to use the GraphDef protobuf serialization -format, since that's understood by both Slim and the rest of TensorFlow. - -``` -! python models/research/slim/export_inference_graph.py \ - --alsologtostderr \ - --dataset_name=visualwakewords \ - --model_name=mobilenet_v1_025 \ - --image_size=96 \ - --input_grayscale=True \ - --output_file=vww_96_grayscale_graph.pb -``` - -If this succeeds, you should have a new 'vww_96_grayscale_graph.pb' file in -your home folder. This contains the layout of the operations in the model, but -doesn't yet have any of the weight data. - -#### Freezing the weights - -The process of storing the trained weights together with the operation graph is -known as freezing. This converts all of the variables in the graph to -constants, after loading their values from a checkpoint file. The command below -uses a checkpoint from the millionth training step, but you can supply any -valid checkpoint path. The graph freezing script is stored inside the main -tensorflow repository, so we have to download this from GitHub before running -this command. - -``` -! git clone https://github.com/tensorflow/tensorflow -! python tensorflow/tensorflow/python/tools/freeze_graph.py \ ---input_graph=vww_96_grayscale_graph.pb \ ---input_checkpoint=vww_96_grayscale/model.ckpt-1000000 \ ---input_binary=true --output_graph=vww_96_grayscale_frozen.pb \ ---output_node_names=MobilenetV1/Predictions/Reshape_1 -``` - -After this, you should see a file called 'vww_96_grayscale_frozen.pb'. - -#### Quantizing and converting to TensorFlow Lite - -Quantization is a tricky and involved process, and it's still very much an -active area of research, so taking the float graph that we've trained so far -and converting it down to eight bit takes quite a bit of code. You can find -more of an explanation of what quantization is and how it works in the chapter -on latency optimization, but here we'll show you how to use it with the model -we've trained. The majority of the code is preparing example images to feed -into the trained network, so that the ranges of the activation layers in -typical use can be measured. We rely on the TFLiteConverter class to handle the -quantization and conversion into the TensorFlow Lite flatbuffer file that we -need for the inference engine. - -``` -import tensorflow as tf -import io -import PIL -import numpy as np - -def representative_dataset_gen(): - - record_iterator = -tf.python_io.tf_record_iterator(path='coco/processed/val.record-00000-of-00010') - - count = 0 - for string_record in record_iterator: - example = tf.train.Example() - example.ParseFromString(string_record) - image_stream = -io.BytesIO(example.features.feature['image/encoded'].bytes_list.value[0]) - image = PIL.Image.open(image_stream) - image = image.resize((96, 96)) - image = image.convert('L') - array = np.array(image) - array = np.expand_dims(array, axis=2) - array = np.expand_dims(array, axis=0) - array = ((array / 127.5) - 1.0).astype(np.float32) - yield([array]) - count += 1 - if count > 300: - break - -converter = -tf.lite.TFLiteConverter.from_frozen_graph('vww_96_grayscale_frozen.pb', -['input'], ['MobilenetV1/Predictions/Reshape_1']) -converter.optimizations = [tf.lite.Optimize.DEFAULT] -converter.representative_dataset = representative_dataset_gen -converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] -converter.inference_input_type = tf.int8 -converter.inference_output_type = tf.int8 - -tflite_quant_model = converter.convert() -open("vww_96_grayscale_quantized.tflite", "wb").write(tflite_quant_model) -``` - -#### Converting into a C source file - -The converter writes out a file, but most embedded devices don't have a file -system. To access the serialized data from our program, we have to compile it -into the executable and store it in Flash. The easiest way to do that is to -convert the file into a C data array. - -``` -# Install xxd if it is not available -!apt-get -qq install xxd -# Save the file as a C source file -!xxd -i vww_96_grayscale_quantized.tflite > person_detect_model_data.cc -``` - -You can now replace the existing person_detect_model_data.cc file with the -version you've trained, and be able to run your own model on embedded devices. - -### Training for other categories - -There are over 60 different object types in the MS-COCO dataset, so an easy way -to customize your model would be to choose one of those instead of 'person' -when you build the training dataset. Here's an example that looks for cars: - -``` -! python models/research/slim/datasets/build_visualwakewords_data.py ---logtostderr \ ---train_image_dir=coco/raw-data/train2014 \ ---val_image_dir=coco/raw-data/val2014 \ ---train_annotations_file=coco/raw-data/annotations/instances_train2014.json \ ---val_annotations_file=coco/raw-data/annotations/instances_val2014.json \ ---output_dir=coco/processed_cars \ ---small_object_area_threshold=0.005 \ ---foreground_class_of_interest='car' -``` - -You should be able to follow the same steps you did for the person detector, -but substitute the new 'coco/processed_cars' path wherever 'coco/processed' -used to be. - -If the kind of object you're interested in isn't present in MS-COCO, you may be -able to use transfer learning to help you train on a custom dataset you've -gathered, even if it's much smaller. We don't have an example of this -yet, but we hope to share one soon. - -### Understanding the architecture - -[MobileNets](https://arxiv.org/abs/1704.04861) are a family of architectures -designed to provide good accuracy for as few weight parameters and arithmetic -operations as possible. There are now multiple versions, but in our case we're -using the original v1 since it required the smallest amount of RAM at runtime. -The core concept behind the architecture is depthwise separable convolution. -This is a variant of classical two-dimensional convolutions that works in a -much more efficient way, without sacrificing very much accuracy. Regular -convolution calculates an output value based on applying a filter of a -particular size across all channels of the input. This means the number of -calculations involved in each output is width of the filter multiplied by -height, multiplied by the number of input channels. Depthwise convolution -breaks this large calculation into separate parts. First each input channel is -filtered by one or more rectangular filters to produce intermediate values. -These values are then combined using pointwise convolutions. This dramatically -reduces the number of calculations needed, and in practice produces similar -results to regular convolution. - -MobileNet v1 is a stack of 14 of these depthwise separable convolution layers -with an average pool, then a fully-connected layer followed by a softmax at the -end. We've specified a 'width multiplier' of 0.25, which has the effect of -reducing the number of computations down to around 60 million per inference, by -shrinking the number of channels in each activation layer by 75% compared to -the standard model. In essence it's very similar to a normal convolutional -neural network in operation, with each layer learning patterns in the input. -Earlier layers act more like edge recognition filters, spotting low-level -structure in the image, and later layers synthesize that information into more -abstract patterns that help with the final object classification. diff --git a/tensorflow/lite/micro/kernels/arc_mli/README.md b/tensorflow/lite/micro/kernels/arc_mli/README.md index 1ded4c84f8a..1995edae9d2 100644 --- a/tensorflow/lite/micro/kernels/arc_mli/README.md +++ b/tensorflow/lite/micro/kernels/arc_mli/README.md @@ -65,7 +65,7 @@ quantization only, `dilation_ratio==1` 3. Average Pooling 4. Max Pooling 5. Fully Connected Currently only -[/tensorflow/lite/micro/examples/person_detection_experimental](/tensorflow/lite/micro/examples/person_detection_experimental) +[/tensorflow/lite/micro/examples/person_detection](/tensorflow/lite/micro/examples/person_detection) is quantized using this specification. Other examples can be executed on ARC-based targets, but will only use reference kernels. diff --git a/tensorflow/lite/micro/tools/make/fix_arduino_subfolders.py b/tensorflow/lite/micro/tools/make/fix_arduino_subfolders.py index 29e388c75e0..0c6d06c15b3 100755 --- a/tensorflow/lite/micro/tools/make/fix_arduino_subfolders.py +++ b/tensorflow/lite/micro/tools/make/fix_arduino_subfolders.py @@ -42,7 +42,7 @@ def move_person_data(library_dir): """Moves the downloaded person model into the examples folder.""" old_person_data_path = os.path.join( library_dir, 'src/tensorflow/lite/micro/tools/make/downloads/' + - 'person_model_grayscale/person_detect_model_data.cpp') + 'person_model_int8/person_detect_model_data.cpp') new_person_data_path = os.path.join( library_dir, 'examples/person_detection/person_detect_model_data.cpp') if os.path.exists(old_person_data_path): @@ -58,28 +58,6 @@ def move_person_data(library_dir): source_file.write(file_contents) -def move_person_data_experimental(library_dir): - """Moves the downloaded person model into the examples folder.""" - old_person_data_path = os.path.join( - library_dir, 'src/tensorflow/lite/micro/tools/make/downloads/' + - 'person_model_int8/person_detect_model_data.cpp') - new_person_data_path = os.path.join( - library_dir, - 'examples/person_detection_experimental/person_detect_model_data.cpp') - if os.path.exists(old_person_data_path): - os.rename(old_person_data_path, new_person_data_path) - # Update include. - with open(new_person_data_path, 'r') as source_file: - file_contents = source_file.read() - file_contents = file_contents.replace( - six.ensure_str( - '#include "tensorflow/lite/micro/examples/' + - 'person_detection_experimental/person_detect_model_data.h"'), - '#include "person_detect_model_data.h"') - with open(new_person_data_path, 'w') as source_file: - source_file.write(file_contents) - - def move_image_data_experimental(library_dir): """Moves the downloaded image detection model into the examples folder.""" old_image_data_path = os.path.join( @@ -117,7 +95,6 @@ def main(unparsed_args): rename_example_subfolder_files(library_dir) rename_example_main_inos(library_dir) move_person_data(library_dir) - move_person_data_experimental(library_dir) move_image_data_experimental(library_dir) diff --git a/tensorflow/lite/micro/tools/make/fix_arduino_subfolders_test.sh b/tensorflow/lite/micro/tools/make/fix_arduino_subfolders_test.sh index 307d026bfa7..e55e61a2329 100755 --- a/tensorflow/lite/micro/tools/make/fix_arduino_subfolders_test.sh +++ b/tensorflow/lite/micro/tools/make/fix_arduino_subfolders_test.sh @@ -30,7 +30,7 @@ mkdir -p `dirname ${EXAMPLES_SUBDIR_HEADER}` touch ${EXAMPLES_SUBDIR_HEADER} TENSORFLOW_SRC_DIR=${LIBRARY_DIR}/src/ -PERSON_DATA_FILE=${TENSORFLOW_SRC_DIR}tensorflow/lite/micro/tools/make/downloads/person_model_grayscale/person_detect_model_data.cpp +PERSON_DATA_FILE=${TENSORFLOW_SRC_DIR}tensorflow/lite/micro/tools/make/downloads/person_model_int8/person_detect_model_data.cpp mkdir -p `dirname ${PERSON_DATA_FILE}` echo '#include "tensorflow/lite/micro/examples/person_detection/person_detect_model_data.h"' > ${PERSON_DATA_FILE} mkdir -p ${LIBRARY_DIR}/examples/person_detection diff --git a/tensorflow/lite/micro/tools/make/targets/stm32f4_makefile.inc b/tensorflow/lite/micro/tools/make/targets/stm32f4_makefile.inc index 2c2785ab256..3bdf1e9bda4 100644 --- a/tensorflow/lite/micro/tools/make/targets/stm32f4_makefile.inc +++ b/tensorflow/lite/micro/tools/make/targets/stm32f4_makefile.inc @@ -74,7 +74,7 @@ MICROLITE_TEST_SRCS := $(filter-out $(EXCLUDED_TESTS), $(MICROLITE_TEST_SRCS)) EXCLUDED_EXAMPLE_TESTS := \ tensorflow/lite/micro/examples/magic_wand/Makefile.inc \ tensorflow/lite/micro/examples/micro_speech/Makefile.inc \ - tensorflow/lite/micro/examples/person_detection_experimental/Makefile.inc \ + tensorflow/lite/micro/examples/person_detection/Makefile.inc \ tensorflow/lite/micro/examples/image_recognition_experimental/Makefile.inc MICRO_LITE_EXAMPLE_TESTS := $(filter-out $(EXCLUDED_EXAMPLE_TESTS), $(MICRO_LITE_EXAMPLE_TESTS)) diff --git a/tensorflow/lite/micro/tools/make/targets/xtensa_makefile.inc b/tensorflow/lite/micro/tools/make/targets/xtensa_makefile.inc index 757408d94cb..f8cafa1d835 100644 --- a/tensorflow/lite/micro/tools/make/targets/xtensa_makefile.inc +++ b/tensorflow/lite/micro/tools/make/targets/xtensa_makefile.inc @@ -56,7 +56,6 @@ EXCLUDED_EXAMPLE_TESTS := \ tensorflow/lite/micro/examples/magic_wand/Makefile.inc \ tensorflow/lite/micro/examples/micro_speech/Makefile.inc \ tensorflow/lite/micro/examples/network_tester/Makefile.inc \ - tensorflow/lite/micro/examples/person_detection/Makefile.inc \ - tensorflow/lite/micro/examples/person_detection_experimental/Makefile.inc + tensorflow/lite/micro/examples/person_detection/Makefile.inc MICRO_LITE_EXAMPLE_TESTS := $(filter-out $(EXCLUDED_EXAMPLE_TESTS), $(MICRO_LITE_EXAMPLE_TESTS)) diff --git a/tensorflow/lite/micro/tools/make/third_party_downloads.inc b/tensorflow/lite/micro/tools/make/third_party_downloads.inc index 7a434ff0f53..7f16f8a77ee 100644 --- a/tensorflow/lite/micro/tools/make/third_party_downloads.inc +++ b/tensorflow/lite/micro/tools/make/third_party_downloads.inc @@ -65,8 +65,8 @@ IMAGE_RECOGNITION_MODEL_MD5 := "1f4607b05ac45b8a6146fb883dbc2d7b" PERSON_MODEL_URL := "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_grayscale_2020_05_27.zip" PERSON_MODEL_MD5 := "55b85f76e2995153e660391d4a209ef1" -PERSON_MODEL_INT8_URL := "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_int8_grayscale_2020_06_23.zip" -PERSON_MODEL_INT8_MD5 := "9b5b6d4677dd0a91b1bb992d1c4c0417" +PERSON_MODEL_INT8_URL := "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_int8_grayscale_2020_12_1.zip" +PERSON_MODEL_INT8_MD5 := "e765cc76889db8640cfe876a37e4ec00" EMBARC_MLI_URL := "https://github.com/foss-for-synopsys-dwc-arc-processors/embarc_mli/archive/ef7dd3c4e37d74a908f30713a7d0121387d3c678.zip" EMBARC_MLI_MD5 := "65c4ff3f4a2963e90fd014f97c69f451"