Create keyword spotting benchmark. Keyword model has scrambled weights, so it will only function as a benchmark for performance optimizations.

PiperOrigin-RevId: 306508532
Change-Id: I056b546fb2e61165132c4c04fc95ec572ec94c48

tensorflow/lite/micro/benchmarks/BUILD

@@ -29,3 +29,25 @@ cc_binary(
         "//tensorflow/lite/micro/testing:micro_test",
     ],
 )
+
+cc_library(
+    name = "keyword_scrambled_model_data",
+    srcs = [
+        "keyword_scrambled_model_data.cc",
+    ],
+    hdrs = [
+        "keyword_scrambled_model_data.h",
+    ],
+)
+
+cc_binary(
+    name = "keyword_benchmark",
+    srcs = ["keyword_benchmark.cc"],
+    deps = [
+        ":keyword_scrambled_model_data",
+        "//tensorflow/lite/c:common",
+        "//tensorflow/lite/micro:micro_framework",
+        "//tensorflow/lite/micro/kernels:micro_ops",
+        "//tensorflow/lite/micro/testing:micro_benchmark",
+    ],
+)

tensorflow/lite/micro/benchmarks/Makefile.inc

@@ -0,0 +1,10 @@
+KEYWORD_BENCHMARK_SRCS := \
+tensorflow/lite/micro/benchmarks/keyword_benchmark.cc \
+tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.cc
+
+KEYWORD_BENCHMARK_HDRS := \
+tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h
+
+# Builds a standalone binary.
+$(eval $(call microlite_test,keyword_benchmark,\
+$(KEYWORD_BENCHMARK_SRCS),$(KEYWORD_BENCHMARK_HDRS)))

tensorflow/lite/micro/benchmarks/README.md

@@ -0,0 +1,33 @@
+# TFLite for Microcontrollers Benchmarks
+
+These benchmarks are for measuring the performance of key models and workloads.
+They are meant to be used as part of the model optimization process for a given
+platform.
+
+## Table of contents
+
+-   [Keyword Benchmark](#keyword-benchmark)
+-   [Run on x86](#run-on-x86)
+-   [Run on Xtensa XPG Simulator](#run-on-xtensa-xpg-simulator)
+
+## Keyword benchmark
+
+The keyword benchmark contains a model for keyword detection with scrambled
+weights and biases.  This model is meant to test performance on a platform only.
+Since the weights are scrambled, the output is meaningless. In order to validate
+the accuracy of optimized kernels, please run the kernel tests.
+
+## Run on x86
+
+To run the keyword benchmark on x86, run
+```
+make -f tensorflow/lite/micro/tools/make/Makefile TAGS=posix test_keyword_benchmark
+```
+
+## Run on Xtensa XPG Simulator
+
+To run the keyword benchmark on the Xtensa XPG simulator, you will need a valid
+Xtensa toolchain and license.  With these set up, run:
+```
+make -f tensorflow/lite/micro/tools/make/Makefile TARGET=xtensa-xpg XTENSA_CORE=<xtensa core>  TAGS=xtensa_hifimini test_keyword_benchmark -j18
+```

tensorflow/lite/micro/benchmarks/keyword_benchmark.cc

@@ -0,0 +1,111 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <cstdint>
+#include <cstdlib>
+
+#include "tensorflow/lite/c/common.h"
+#include "tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h"
+#include "tensorflow/lite/micro/kernels/micro_ops.h"
+#include "tensorflow/lite/micro/micro_error_reporter.h"
+#include "tensorflow/lite/micro/micro_interpreter.h"
+#include "tensorflow/lite/micro/micro_mutable_op_resolver.h"
+#include "tensorflow/lite/micro/testing/micro_benchmark.h"
+
+/*
+ * Keyword Spotting Benchmark for performance optimizations. The model used in
+ * this benchmark only serves as a reference. The values assigned to the model
+ * weights and parameters are not representative of the original model.
+ */
+
+namespace {
+
+// Create an area of memory to use for input, output, and intermediate arrays.
+constexpr int tensor_arena_size = 73 * 1024;
+uint8_t tensor_arena[tensor_arena_size];
+// A random number generator seed to generate input values.
+constexpr int kRandomSeed = 42;
+
+class KeywordRunner {
+ public:
+  KeywordRunner()
+      : keyword_spotting_model_(
+            tflite::GetModel(g_keyword_scrambled_model_data)),
+        reporter_(&micro_reporter_),
+        interpreter_(keyword_spotting_model_, resolver_, tensor_arena,
+                     tensor_arena_size, reporter_) {
+    resolver_.AddBuiltin(tflite::BuiltinOperator_SVDF,
+                         tflite::ops::micro::Register_SVDF());
+    resolver_.AddBuiltin(tflite::BuiltinOperator_FULLY_CONNECTED,
+                         tflite::ops::micro::Register_FULLY_CONNECTED());
+    resolver_.AddBuiltin(tflite::BuiltinOperator_QUANTIZE,
+                         tflite::ops::micro::Register_QUANTIZE());
+    resolver_.AddBuiltin(tflite::BuiltinOperator_DEQUANTIZE,
+                         tflite::ops::micro::Register_DEQUANTIZE(), 1, 2);
+    resolver_.AddBuiltin(tflite::BuiltinOperator_SOFTMAX,
+                         tflite::ops::micro::Register_SOFTMAX());
+    interpreter_.AllocateTensors();
+
+    // The pseudo-random number generator is initialized to a constant seed
+    std::srand(kRandomSeed);
+    TfLiteTensor* input = interpreter_.input(0);
+    TFLITE_CHECK_EQ(input->type, kTfLiteInt16);
+
+    // Pre-populate input tensor with random values.
+    int input_length = input->bytes / sizeof(int16_t);
+    int16_t* input_values = tflite::GetTensorData<int16_t>(input);
+    for (int i = 0; i < input_length; i++) {
+      // Pre-populate input tensor with a random value based on a constant seed.
+      input_values[i] = static_cast<int16_t>(std::rand() % INT16_MAX);
+    }
+  }
+
+  void RunSingleIteration() {
+    // Run the model on this input and make sure it succeeds.
+    TfLiteStatus invoke_status = interpreter_.Invoke();
+    if (invoke_status != kTfLiteOk) {
+      TF_LITE_REPORT_ERROR(reporter_, "Invoke failed.");
+    }
+  }
+
+ private:
+  const tflite::Model* keyword_spotting_model_;
+  tflite::MicroErrorReporter micro_reporter_;
+  tflite::ErrorReporter* reporter_;
+  tflite::MicroOpResolver<6> resolver_;
+  tflite::MicroInterpreter interpreter_;
+};
+
+// NOLINTNEXTLINE
+KeywordRunner runner;
+
+void KeywordRunFirstIteration() { runner.RunSingleIteration(); }
+
+void KeywordRunTenIerations() {
+  // TODO(b/152644476): Add a way to run more than a single deterministic input.
+  for (int i = 0; i < 10; i++) {
+    runner.RunSingleIteration();
+  }
+}
+
+}  //  namespace
+
+TF_LITE_MICRO_BENCHMARKS_BEGIN
+
+TF_LITE_MICRO_BENCHMARK(KeywordRunFirstIteration);
+
+TF_LITE_MICRO_BENCHMARK(KeywordRunTenIerations);
+
+TF_LITE_MICRO_BENCHMARKS_END

tensorflow/lite/micro/benchmarks/keyword_scrambled_model_data.h

@@ -0,0 +1,22 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_BENCHMARKS_KEYWORD_SCRAMBLED_MODEL_DATA_H_
+#define TENSORFLOW_LITE_MICRO_BENCHMARKS_KEYWORD_SCRAMBLED_MODEL_DATA_H_
+
+extern const unsigned char g_keyword_scrambled_model_data[];
+extern const unsigned int g_keyword_scrambled_model_data_length;
+
+#endif  // TENSORFLOW_LITE_MICRO_BENCHMARKS_KEYWORD_SCRAMBLED_MODEL_DATA_H_

tensorflow/lite/micro/testing/micro_benchmark.h

@@ -38,6 +38,9 @@ extern tflite::ErrorReporter* reporter;
 #define TF_LITE_MICRO_BENCHMARKS_END }
 
 #define TF_LITE_MICRO_BENCHMARK(func)                                         \
+  if (tflite::ticks_per_second() == 0) {                                      \
+    return 0;                                                                 \
+  }                                                                           \
   start_ticks = tflite::GetCurrentTimeTicks();                                \
   func();                                                                     \
   duration_ticks = tflite::GetCurrentTimeTicks() - start_ticks;               \

tensorflow/lite/micro/tools/make/Makefile

@@ -96,13 +96,16 @@ MICROLITE_LIB_NAME := libtensorflow-microlite.a
 
 MICRO_LITE_EXAMPLE_TESTS := $(wildcard tensorflow/lite/micro/examples/*/Makefile.inc)
 
+MICRO_LITE_BENCHMARKS := $(wildcard tensorflow/lite/micro/benchmarks/Makefile.inc)
+
 MICROLITE_TEST_SRCS := \
 $(wildcard tensorflow/lite/micro/*test.cc) \
 $(wildcard tensorflow/lite/micro/kernels/*test.cc) \
 $(wildcard tensorflow/lite/micro/memory_planner/*test.cc)
 
+# TODO(b/152645559): move all benchmarks to benchmarks directory.
 MICROLITE_BENCHMARK_SRCS := \
-$(wildcard tensorflow/lite/micro/kernels/benchmarks/*.cc)
+$(wildcard tensorflow/lite/micro/benchmarks/*.cc)
 
 MICROLITE_TEST_HDRS := \
 $(wildcard tensorflow/lite/micro/testing/*.h)
@@ -266,6 +269,9 @@ AR := $(TARGET_TOOLCHAIN_ROOT)${TARGET_TOOLCHAIN_PREFIX}${AR_TOOL}
 # Load the examples.
 include $(MICRO_LITE_EXAMPLE_TESTS)
 
+# Load the benchmarks.
+include $(MICRO_LITE_BENCHMARKS)
+
 # Create rules for downloading third-party dependencies.
 THIRD_PARTY_TARGETS :=
 $(foreach DOWNLOAD,$(THIRD_PARTY_DOWNLOADS),$(eval $(call create_download_rule,$(DOWNLOAD))))

tensorflow/lite/micro/xtensa-xpg/micro_time.cc

@@ -0,0 +1,28 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Xtensa implementation of micro_timer.
+// To include this with make, add TAGS=xtensa-xpg.
+#include "tensorflow/lite/micro/micro_time.h"
+
+#include <time.h>
+
+namespace tflite {
+
+int32_t ticks_per_second() { return CLOCKS_PER_SEC; }
+
+int32_t GetCurrentTimeTicks() { return clock(); }
+
+}  // namespace tflite