Create a micro profiler class and use it to implement per op profiling in the micro interpreter.

PiperOrigin-RevId: 316202980
Change-Id: Ifc7e238835a2458d17d8c0917ba2d33b05af8c96

tensorflow/lite/micro/BUILD

@@ -48,6 +48,7 @@ cc_library(
         "//tensorflow/lite/core/api",
         "//tensorflow/lite/kernels/internal:compatibility",
         "//tensorflow/lite/kernels/internal:tensor",
+        "//tensorflow/lite/micro:micro_profiler",
         "//tensorflow/lite/micro/memory_planner",
         "//tensorflow/lite/micro/memory_planner:greedy_memory_planner",
         "//tensorflow/lite/schema:schema_fbs",
@@ -181,6 +182,24 @@ cc_library(
     deps = ["//tensorflow/lite/c:common"],
 )
 
+cc_library(
+    name = "micro_profiler",
+    srcs = [
+        "micro_profiler.cc",
+    ],
+    hdrs = [
+        "micro_profiler.h",
+    ],
+    build_for_embedded = True,
+    copts = micro_copts(),
+    deps = [
+        ":micro_compatibility",
+        ":micro_time",
+        "//tensorflow/lite/core/api",
+        "//tensorflow/lite/kernels/internal:compatibility",
+    ],
+)
+
 cc_library(
     name = "micro_utils",
     srcs = [

tensorflow/lite/micro/micro_interpreter.cc

@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/lite/core/api/tensor_utils.h"
 #include "tensorflow/lite/micro/micro_allocator.h"
 #include "tensorflow/lite/micro/micro_op_resolver.h"
+#include "tensorflow/lite/micro/micro_profiler.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 
 namespace tflite {
@@ -75,26 +76,28 @@ MicroInterpreter::MicroInterpreter(const Model* model,
                                    const MicroOpResolver& op_resolver,
                                    uint8_t* tensor_arena,
                                    size_t tensor_arena_size,
-                                   ErrorReporter* error_reporter)
+                                   ErrorReporter* error_reporter,
+                                   tflite::Profiler* profiler)
     : model_(model),
       op_resolver_(op_resolver),
       error_reporter_(error_reporter),
       allocator_(*MicroAllocator::Create(tensor_arena, tensor_arena_size,
                                          error_reporter)),
       context_helper_(error_reporter_, &allocator_) {
-  Init();
+  Init(profiler);
 }
 
 MicroInterpreter::MicroInterpreter(const Model* model,
                                    const MicroOpResolver* op_resolver,
                                    MicroAllocator* allocator,
-                                   ErrorReporter* error_reporter)
+                                   ErrorReporter* error_reporter,
+                                   tflite::Profiler* profiler)
     : model_(model),
       op_resolver_(*op_resolver),
       error_reporter_(error_reporter),
       allocator_(*allocator),
       context_helper_(error_reporter_, &allocator_) {
-  Init();
+  Init(profiler);
 }
 
 MicroInterpreter::~MicroInterpreter() {
@@ -112,7 +115,7 @@ MicroInterpreter::~MicroInterpreter() {
   }
 }
 
-void MicroInterpreter::Init() {
+void MicroInterpreter::Init(tflite::Profiler* profiler) {
   const flatbuffers::Vector<flatbuffers::Offset<SubGraph>>* subgraphs =
       model_->subgraphs();
   if (subgraphs->size() != 1) {
@@ -126,6 +129,7 @@ void MicroInterpreter::Init() {
   context_.impl_ = static_cast<void*>(&context_helper_);
   context_.ReportError = context_helper_.ReportOpError;
   context_.recommended_num_threads = 1;
+  context_.profiler = profiler;
 
   initialization_status_ = kTfLiteOk;
 }
@@ -266,7 +270,16 @@ TfLiteStatus MicroInterpreter::Invoke() {
     auto* registration = node_and_registrations_[i].registration;
 
     if (registration->invoke) {
-      TfLiteStatus invoke_status = registration->invoke(&context_, node);
+      TfLiteStatus invoke_status;
+#ifndef NDEBUG  // Omit profiler overhead from release builds.
+      // The case where profiler == nullptr is handled by ScopedOperatorProfile.
+      tflite::Profiler* profiler =
+          reinterpret_cast<tflite::Profiler*>(context_.profiler);
+      ScopedOperatorProfile scoped_profiler(
+          profiler, OpNameFromRegistration(registration), i);
+#endif
+      invoke_status = registration->invoke(&context_, node);
+
       if (invoke_status == kTfLiteError) {
         TF_LITE_REPORT_ERROR(
             error_reporter_,

tensorflow/lite/micro/micro_interpreter.h

@@ -21,6 +21,7 @@ limitations under the License.
 #include "flatbuffers/flatbuffers.h"  // from @flatbuffers
 #include "tensorflow/lite/c/common.h"
 #include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/api/profiler.h"
 #include "tensorflow/lite/kernels/internal/tensor_ctypes.h"
 #include "tensorflow/lite/micro/micro_allocator.h"
 #include "tensorflow/lite/micro/micro_op_resolver.h"
@@ -64,17 +65,17 @@ class ContextHelper {
 
 class MicroInterpreter {
  public:
-  // The lifetime of the model, op resolver, tensor arena, and error reporter
+  // The lifetime of the model, op resolver, tensor arena, error reporter and
-  // must be at least as long as that of the interpreter object, since the
+  // profiler must be at least as long as that of the interpreter object, since
-  // interpreter may need to access them at any time. This means that you should
+  // the interpreter may need to access them at any time. This means that you
-  // usually create them with the same scope as each other, for example having
+  // should usually create them with the same scope as each other, for example
-  // them all allocated on the stack as local variables through a top-level
+  // having them all allocated on the stack as local variables through a
-  // function.
+  // top-level function. The interpreter doesn't do any deallocation of any of
-  // The interpreter doesn't do any deallocation of any of the pointed-to
+  // the pointed-to objects, ownership remains with the caller.
-  // objects, ownership remains with the caller.
   MicroInterpreter(const Model* model, const MicroOpResolver& op_resolver,
                    uint8_t* tensor_arena, size_t tensor_arena_size,
-                   ErrorReporter* error_reporter);
+                   ErrorReporter* error_reporter,
+                   tflite::Profiler* profiler = nullptr);
 
   // Create an interpreter instance using an existing MicroAllocator instance.
   // This constructor should be used when creating an allocator that needs to
@@ -82,7 +83,8 @@ class MicroInterpreter {
   // allocations inside the interpreter. The lifetime of the allocator must be
   // as long as that of the interpreter object.
   MicroInterpreter(const Model* model, const MicroOpResolver* op_resolver,
-                   MicroAllocator* allocator, ErrorReporter* error_reporter);
+                   MicroAllocator* allocator, ErrorReporter* error_reporter,
+                   tflite::Profiler* profiler = nullptr);
 
   ~MicroInterpreter();
 
@@ -166,7 +168,7 @@ class MicroInterpreter {
  private:
   // TODO(b/158263161): Consider switching to Create() function to enable better
   // error reporting during initialization.
-  void Init();
+  void Init(tflite::Profiler* profiler);
 
   void CorrectTensorEndianness(TfLiteTensor* tensorCorr);
 

tensorflow/lite/micro/micro_interpreter_test.cc

@@ -24,6 +24,45 @@ limitations under the License.
 #include "tensorflow/lite/micro/test_helpers.h"
 #include "tensorflow/lite/micro/testing/micro_test.h"
 
+namespace tflite {
+namespace {
+
+class MockProfiler : public tflite::Profiler {
+ public:
+  MockProfiler() : event_starts_(0), event_ends_(0) {}
+  ~MockProfiler() override = default;
+
+  // AddEvent is unused for Tf Micro.
+  void AddEvent(const char* tag, EventType event_type, uint64_t start,
+                uint64_t end, int64_t event_metadata1,
+                int64_t event_metadata2) override{};
+
+  // BeginEvent followed by code followed by EndEvent will profile the code
+  // enclosed. Multiple concurrent events are unsupported, so the return value
+  // is always 0. Event_metadata1 and event_metadata2 are unused. The tag
+  // pointer must be valid until EndEvent is called.
+  uint32_t BeginEvent(const char* tag, EventType event_type,
+                      int64_t event_metadata1,
+                      int64_t event_metadata2) override {
+    event_starts_++;
+    return 0;
+  }
+
+  // Event_handle is ignored since TF Micro does not support concurrent events.
+  void EndEvent(uint32_t event_handle) override { event_ends_++; }
+
+  int event_starts() { return event_starts_; }
+  int event_ends() { return event_ends_; }
+
+ private:
+  int event_starts_;
+  int event_ends_;
+  TF_LITE_REMOVE_VIRTUAL_DELETE
+};
+
+}  // namespace
+}  // namespace tflite
+
 TF_LITE_MICRO_TESTS_BEGIN
 
 TF_LITE_MICRO_TEST(TestInterpreter) {
@@ -210,4 +249,31 @@ TF_LITE_MICRO_TEST(TestIncompleteInitialization) {
                                        micro_test::reporter);
 }
 
+// Test that an interpreter with a supplied profiler correctly calls the
+// profiler each time an operator is invoked.
+TF_LITE_MICRO_TEST(InterpreterWithProfilerShouldProfileOps) {
+  const tflite::Model* model = tflite::testing::GetComplexMockModel();
+  TF_LITE_MICRO_EXPECT_NE(nullptr, model);
+
+  tflite::testing::MockOpResolver mock_resolver;
+  constexpr size_t allocator_buffer_size = 2048;
+  uint8_t allocator_buffer[allocator_buffer_size];
+  tflite::MockProfiler profiler;
+  tflite::MicroInterpreter interpreter(model, mock_resolver, allocator_buffer,
+                                       allocator_buffer_size,
+                                       micro_test::reporter, &profiler);
+
+  TF_LITE_MICRO_EXPECT_EQ(profiler.event_starts(), 0);
+  TF_LITE_MICRO_EXPECT_EQ(profiler.event_ends(), 0);
+  TF_LITE_MICRO_EXPECT_EQ(interpreter.AllocateTensors(), kTfLiteOk);
+  TF_LITE_MICRO_EXPECT_EQ(interpreter.Invoke(), kTfLiteOk);
+#ifndef NDEBUG
+  TF_LITE_MICRO_EXPECT_EQ(profiler.event_starts(), 3);
+  TF_LITE_MICRO_EXPECT_EQ(profiler.event_ends(), 3);
+#else  // Profile events will not occur on release builds.
+  TF_LITE_MICRO_EXPECT_EQ(profiler.event_starts(), 0);
+  TF_LITE_MICRO_EXPECT_EQ(profiler.event_ends(), 0);
+#endif
+}
+
 TF_LITE_MICRO_TESTS_END

tensorflow/lite/micro/micro_profiler.cc

@@ -0,0 +1,41 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/micro/micro_profiler.h"
+
+#include "tensorflow/lite/kernels/internal/compatibility.h"
+#include "tensorflow/lite/micro/micro_time.h"
+
+namespace tflite {
+
+MicroProfiler::MicroProfiler(tflite::ErrorReporter* reporter)
+    : reporter_(reporter) {}
+
+uint32_t MicroProfiler::BeginEvent(const char* tag, EventType event_type,
+                                   int64_t event_metadata1,
+                                   int64_t event_metadata2) {
+  start_time_ = GetCurrentTimeTicks();
+  TFLITE_DCHECK(tag != nullptr);
+  event_tag_ = tag;
+  return 0;
+}
+
+void MicroProfiler::EndEvent(uint32_t event_handle) {
+  int32_t end_time = GetCurrentTimeTicks();
+  TF_LITE_REPORT_ERROR(reporter_, "%s took %d cycles\n", event_tag_,
+                       end_time - start_time_);
+}
+
+}  // namespace tflite

tensorflow/lite/micro/micro_profiler.h

@@ -0,0 +1,71 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_MICRO_MICRO_PROFILER_H_
+#define TENSORFLOW_LITE_MICRO_MICRO_PROFILER_H_
+
+#include "tensorflow/lite/core/api/error_reporter.h"
+#include "tensorflow/lite/core/api/profiler.h"
+#include "tensorflow/lite/micro/compatibility.h"
+
+namespace tflite {
+
+// MicroProfiler creates a common way to gain fine-grained insight into runtime
+// performance. Bottleck operators can be identified along with slow code
+// sections. This can be used in conjunction with running the relevant micro
+// benchmark to evaluate end-to-end performance.
+//
+// Usage example:
+// MicroProfiler profiler(error_reporter);
+// {
+//   ScopedProfile scoped_profile(profiler, tag);
+//   work_to_profile();
+// }
+//
+// This will call the following methods in order:
+// int event_handle = profiler->BeginEvent(op_name, EventType::DEFAULT, 0)
+// work_to_profile();
+// profiler->EndEvent(event_handle)
+class MicroProfiler : public tflite::Profiler {
+ public:
+  explicit MicroProfiler(tflite::ErrorReporter* reporter);
+  ~MicroProfiler() override = default;
+
+  // AddEvent is unused for Tf Micro.
+  void AddEvent(const char* tag, EventType event_type, uint64_t start,
+                uint64_t end, int64_t event_metadata1,
+                int64_t event_metadata2) override{};
+
+  // BeginEvent followed by code followed by EndEvent will profile the code
+  // enclosed. Multiple concurrent events are unsupported, so the return value
+  // is always 0. Event_metadata1 and event_metadata2 are unused. The tag
+  // pointer must be valid until EndEvent is called.
+  uint32_t BeginEvent(const char* tag, EventType event_type,
+                      int64_t event_metadata1,
+                      int64_t event_metadata2) override;
+
+  // Event_handle is ignored since TF Micro does not support concurrent events.
+  void EndEvent(uint32_t event_handle) override;
+
+ private:
+  tflite::ErrorReporter* reporter_;
+  int32_t start_time_;
+  const char* event_tag_;
+  TF_LITE_REMOVE_VIRTUAL_DELETE
+};
+
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_MICRO_MICRO_PROFILER_H_