Support benchmarking a set of TFLite performance options in a single shot.

PiperOrigin-RevId: 260076670

tensorflow/lite/tools/benchmark/BUILD

@@ -36,6 +36,26 @@ cc_binary(
     ],
 )
 
+cc_binary(
+    name = "benchmark_model_performance_options",
+    srcs = [
+        "benchmark_tflite_performance_options_main.cc",
+    ],
+    copts = common_copts,
+    linkopts = tflite_linkopts() + select({
+        "//tensorflow:android": [
+            "-pie",  # Android 5.0 and later supports only PIE
+            "-lm",  # some builtin ops, e.g., tanh, need -lm
+        ],
+        "//conditions:default": [],
+    }),
+    deps = [
+        ":benchmark_performance_options",
+        ":benchmark_tflite_model_lib",
+        ":logging",
+    ],
+)
+
 tf_cc_binary(
     name = "benchmark_model_plus_flex",
     srcs = [
@@ -99,6 +119,22 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "benchmark_performance_options",
+    srcs = [
+        "benchmark_performance_options.cc",
+    ],
+    hdrs = ["benchmark_performance_options.h"],
+    copts = common_copts,
+    deps = [
+        ":benchmark_model_lib",
+        ":benchmark_utils",
+        ":logging",
+        "//tensorflow/lite/profiling:time",
+        "//tensorflow/lite/tools:command_line_flags",
+    ],
+)
+
 cc_library(
     name = "benchmark_params",
     srcs = [

tensorflow/lite/tools/benchmark/README.md

@@ -213,3 +213,19 @@ Memory (bytes): count=0
 
 Average inference timings in us: Warmup: 83235, Init: 38467, no stats: 79760.9
 ```
+
+## Benchmark multiple performance options in a single run
+
+A convenient and simple C++ binary is also provided to benchmark multiple
+performance options in a single run. This binary is built based on the
+aforementioned benchmark tool that could only benchmark a single performance
+option at a time. They share the same build/install/run process, but the BUILD
+target name of this binary is `benchmark_model_performance_options` and it takes
+some additional parameters as detailed below.
+
+### Additional Parameters
+*   `perf_options_list`: `string` (default='all') \
+    A comma-separated list of TFLite performance options to benchmark.
+*   `option_benchmark_run_delay`: `float` (default=-1.0) \
+    The delay between two consecutive runs of benchmarking performance options
+    in seconds.

tensorflow/lite/tools/benchmark/benchmark_model.cc

@@ -42,7 +42,7 @@ BenchmarkParams BenchmarkModel::DefaultParams() {
 
 BenchmarkModel::BenchmarkModel() : params_(DefaultParams()) {}
 
-void BenchmarkLoggingListener::OnBenchmarkEnd(const BenchmarkResults &results) {
+void BenchmarkLoggingListener::OnBenchmarkEnd(const BenchmarkResults& results) {
   auto inference_us = results.inference_time_us();
   auto init_us = results.startup_latency_us();
   auto warmup_us = results.warmup_time_us();
@@ -143,7 +143,7 @@ Stat<int64_t> BenchmarkModel::Run(int min_num_times, float min_secs,
 
 bool BenchmarkModel::ValidateParams() { return true; }
 
-void BenchmarkModel::Run(int argc, char **argv) {
+void BenchmarkModel::Run(int argc, char** argv) {
   if (!ParseFlags(argc, argv)) {
     return;
   }
@@ -174,10 +174,10 @@ void BenchmarkModel::Run() {
       {startup_latency_us, input_bytes, warmup_time_us, inference_time_us});
 }
 
-bool BenchmarkModel::ParseFlags(int argc, char **argv) {
+bool BenchmarkModel::ParseFlags(int* argc, char** argv) {
   auto flag_list = GetFlags();
   const bool parse_result =
-      Flags::Parse(&argc, const_cast<const char **>(argv), flag_list);
+      Flags::Parse(argc, const_cast<const char**>(argv), flag_list);
   if (!parse_result) {
     std::string usage = Flags::Usage(argv[0], flag_list);
     TFLITE_LOG(ERROR) << usage;

tensorflow/lite/tools/benchmark/benchmark_model.h

@@ -129,8 +129,9 @@ class BenchmarkLoggingListener : public BenchmarkListener {
 template <typename T>
 Flag CreateFlag(const char* name, BenchmarkParams* params,
                 const std::string& usage) {
-  return Flag(name, [params, name](const T& val) { params->Set<T>(name, val); },
+  return Flag(
-              params->Get<T>(name), usage);
+      name, [params, name](const T& val) { params->Set<T>(name, val); },
+      params->Get<T>(name), usage);
 }
 
 // Benchmarks a model.
@@ -150,11 +151,19 @@ class BenchmarkModel {
     listeners_.AddListener(listener);
   }
 
+  BenchmarkParams* mutable_params() { return &params_; }
+
+  // Unparsable flags will remain in 'argv' in the original order and 'argc'
+  // will be updated accordingly.
+  bool ParseFlags(int* argc, char** argv);
+
  protected:
   virtual void LogParams();
   virtual bool ValidateParams();
-  bool ParseFlags(int argc, char** argv);
+
+  bool ParseFlags(int argc, char** argv) { return ParseFlags(&argc, argv); }
   virtual std::vector<Flag> GetFlags();
+
   virtual uint64_t ComputeInputBytes() = 0;
   virtual tensorflow::Stat<int64_t> Run(int min_num_times, float min_secs,
                                         float max_secs, RunType run_type);

tensorflow/lite/tools/benchmark/benchmark_performance_options.cc

@@ -0,0 +1,175 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/tools/benchmark/benchmark_performance_options.h"
+
+#include <algorithm>
+
+#include "tensorflow/lite/profiling/time.h"
+#include "tensorflow/lite/tools/benchmark/benchmark_utils.h"
+#include "tensorflow/lite/tools/benchmark/logging.h"
+#include "tensorflow/lite/tools/command_line_flags.h"
+
+namespace tflite {
+namespace benchmark {
+
+BenchmarkParams BenchmarkPerformanceOptions::DefaultParams() {
+  BenchmarkParams params;
+  params.AddParam("perf_options_list",
+                  BenchmarkParam::Create<std::string>("all"));
+  params.AddParam("option_benchmark_run_delay",
+                  BenchmarkParam::Create<float>(-1.0f));
+  return params;
+}
+
+std::vector<Flag> BenchmarkPerformanceOptions::GetFlags() {
+  return {
+      CreateFlag<std::string>(
+          "perf_options_list", &params_,
+          "A comma-separated list of TFLite performance options to benchmark. "
+          "By default, all performance options are benchmarked."),
+      CreateFlag<float>("option_benchmark_run_delay", &params_,
+                        "The delay between two consecutive runs of "
+                        "benchmarking performance options in seconds."),
+  };
+}
+
+bool BenchmarkPerformanceOptions::ParseFlags(int* argc, char** argv) {
+  auto flag_list = GetFlags();
+  const bool parse_result =
+      Flags::Parse(argc, const_cast<const char**>(argv), flag_list);
+  if (!parse_result) {
+    std::string usage = Flags::Usage(argv[0], flag_list);
+    TFLITE_LOG(ERROR) << usage;
+    return false;
+  }
+
+  // Parse the value of --perf_options_list to find performance options to be
+  // benchmarked.
+  return ParsePerfOptions();
+}
+
+bool BenchmarkPerformanceOptions::ParsePerfOptions() {
+  const auto& perf_options_list = params_.Get<std::string>("perf_options_list");
+  if (!util::SplitAndParse(perf_options_list, ',', &perf_options_)) {
+    TFLITE_LOG(ERROR) << "Cannot parse --perf_options_list: '"
+                      << perf_options_list
+                      << "'. Please double-check its value.";
+    perf_options_.clear();
+    return false;
+  }
+
+  const auto valid_options = GetValidPerfOptions();
+  bool is_valid = true;
+  for (const auto& option : perf_options_) {
+    if (std::find(valid_options.begin(), valid_options.end(), option) ==
+        valid_options.end()) {
+      is_valid = false;
+      break;
+    }
+  }
+  if (!is_valid) {
+    std::string valid_options_str;
+    for (int i = 0; i < valid_options.size() - 1; ++i) {
+      valid_options_str += (valid_options[i] + ", ");
+    }
+    valid_options_str += valid_options.back();
+    TFLITE_LOG(ERROR)
+        << "There are invalid perf options in --perf_options_list: '"
+        << perf_options_list << "'. Valid perf options are: ["
+        << valid_options_str << "]";
+    perf_options_.clear();
+    return false;
+  }
+  return true;
+}
+
+std::vector<std::string> BenchmarkPerformanceOptions::GetValidPerfOptions()
+    const {
+  return {"all", "cpu", "gpu", "nnapi"};
+}
+
+bool BenchmarkPerformanceOptions::HasOption(const string& option) const {
+  return std::find(perf_options_.begin(), perf_options_.end(), option) !=
+         perf_options_.end();
+}
+
+void BenchmarkPerformanceOptions::ResetPerformanceOptions() {
+  single_option_run_params_->Set<int32_t>("num_threads", 1);
+  single_option_run_params_->Set<bool>("use_gpu", false);
+  single_option_run_params_->Set<bool>("use_nnapi", false);
+}
+
+void BenchmarkPerformanceOptions::BenchmarkCPUOptions() {
+  // Reset all performance-related options before any runs.
+  ResetPerformanceOptions();
+
+  const int num_threads[] = {1, 2, 4};
+  for (int i = 0; i < sizeof(num_threads) / sizeof(int); ++i) {
+    single_option_run_params_->Set<int32_t>("num_threads", num_threads[i]);
+    util::SleepForSeconds(params_.Get<float>("option_benchmark_run_delay"));
+    single_option_run_->Run();
+  }
+}
+
+void BenchmarkPerformanceOptions::BenchmarkGPUOptions() {
+  // Reset all performance-related options before any runs.
+  ResetPerformanceOptions();
+
+  single_option_run_params_->Set<bool>("use_gpu", true);
+  util::SleepForSeconds(params_.Get<float>("option_benchmark_run_delay"));
+  single_option_run_->Run();
+}
+
+void BenchmarkPerformanceOptions::BenchmarkNnapiOptions() {
+  // Reset all performance-related options before any runs.
+  ResetPerformanceOptions();
+
+  single_option_run_params_->Set<bool>("use_nnapi", true);
+  util::SleepForSeconds(params_.Get<float>("option_benchmark_run_delay"));
+  single_option_run_->Run();
+}
+
+void BenchmarkPerformanceOptions::Run(int argc, char** argv) {
+  // We first parse flags for single-option runs to get information like
+  // parameters of the input model etc.
+  if (!single_option_run_->ParseFlags(&argc, argv)) {
+    return;
+  }
+
+  // Now, we parse flags that are specified for this particular binary.
+  if (!ParseFlags(&argc, argv)) {
+    return;
+  }
+
+  TFLITE_LOG(INFO) << "The list of TFLite runtime options to be benchmarked: ["
+                   << params_.Get<std::string>("perf_options_list") << "]";
+
+  const bool benchmark_all = HasOption("all");
+  if (benchmark_all || HasOption("cpu")) {
+    BenchmarkCPUOptions();
+  }
+
+  if (benchmark_all || HasOption("gpu")) {
+    BenchmarkGPUOptions();
+  }
+
+  if (benchmark_all || HasOption("nnapi")) {
+    BenchmarkNnapiOptions();
+  }
+}
+
+}  // namespace benchmark
+}  // namespace tflite

tensorflow/lite/tools/benchmark/benchmark_performance_options.h

@@ -0,0 +1,70 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_
+#define TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_
+
+#include "tensorflow/lite/tools/benchmark/benchmark_model.h"
+
+namespace tflite {
+namespace benchmark {
+
+// Benchmarks all performance options on a model by repeatedly invoking the
+// single-performance-option run on a passed-in 'BenchmarkModel' object.
+class BenchmarkPerformanceOptions {
+ public:
+  // Doesn't own the memory of 'single_option_run'.
+  explicit BenchmarkPerformanceOptions(BenchmarkModel* single_option_run)
+      : BenchmarkPerformanceOptions(DefaultParams(), single_option_run) {}
+
+  BenchmarkPerformanceOptions(BenchmarkParams params,
+                              BenchmarkModel* single_option_run)
+      : params_(std::move(params)),
+        single_option_run_(single_option_run),
+        single_option_run_params_(single_option_run->mutable_params()) {}
+
+  virtual ~BenchmarkPerformanceOptions() {}
+
+  virtual void Run(int argc, char** argv);
+
+ protected:
+  static BenchmarkParams DefaultParams();
+
+  // Unparsable flags will remain in 'argv' in the original order and 'argc'
+  // will be updated accordingly.
+  bool ParseFlags(int* argc, char** argv);
+  virtual std::vector<Flag> GetFlags();
+
+  bool ParsePerfOptions();
+  virtual std::vector<std::string> GetValidPerfOptions() const;
+  bool HasOption(const string& option) const;
+  virtual void ResetPerformanceOptions();
+
+  virtual void BenchmarkCPUOptions();
+  virtual void BenchmarkGPUOptions();
+  virtual void BenchmarkNnapiOptions();
+
+  BenchmarkParams params_;
+  std::vector<std::string> perf_options_;
+
+  // The object that drives a single-performance-option run.
+  BenchmarkModel* const single_option_run_;          // Doesn't own the memory.
+  BenchmarkParams* const single_option_run_params_;  // Doesn't own the memory.
+};
+
+}  // namespace benchmark
+}  // namespace tflite
+
+#endif  // TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_

tensorflow/lite/tools/benchmark/benchmark_tflite_performance_options_main.cc

@@ -0,0 +1,40 @@
+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/lite/tools/benchmark/benchmark_performance_options.h"
+#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h"
+#include "tensorflow/lite/tools/benchmark/logging.h"
+
+namespace tflite {
+namespace benchmark {
+
+int Main(int argc, char** argv) {
+#ifdef TFLITE_CUSTOM_OPS_HEADER
+  TFLITE_LOG(INFO) << "STARTING with custom ops!";
+#else
+  TFLITE_LOG(INFO) << "STARTING!";
+#endif
+  BenchmarkTfLiteModel benchmark;
+  BenchmarkLoggingListener listener;
+  benchmark.AddListener(&listener);
+
+  BenchmarkPerformanceOptions all_options_benchmark(&benchmark);
+  all_options_benchmark.Run(argc, argv);
+  return EXIT_SUCCESS;
+}
+}  // namespace benchmark
+}  // namespace tflite
+
+int main(int argc, char** argv) { return tflite::benchmark::Main(argc, argv); }