From ea9eea1681c317c663318ebed7fb9900043425ed Mon Sep 17 00:00:00 2001 From: Chao Mei Date: Thu, 25 Jul 2019 20:52:41 -0700 Subject: [PATCH] Support benchmarking a set of TFLite performance options in a single shot. PiperOrigin-RevId: 260076670 --- tensorflow/lite/tools/benchmark/BUILD | 36 ++++ tensorflow/lite/tools/benchmark/README.md | 16 ++ .../lite/tools/benchmark/benchmark_model.cc | 8 +- .../lite/tools/benchmark/benchmark_model.h | 15 +- .../benchmark_performance_options.cc | 175 ++++++++++++++++++ .../benchmark/benchmark_performance_options.h | 70 +++++++ ...nchmark_tflite_performance_options_main.cc | 40 ++++ 7 files changed, 353 insertions(+), 7 deletions(-) create mode 100644 tensorflow/lite/tools/benchmark/benchmark_performance_options.cc create mode 100644 tensorflow/lite/tools/benchmark/benchmark_performance_options.h create mode 100644 tensorflow/lite/tools/benchmark/benchmark_tflite_performance_options_main.cc diff --git a/tensorflow/lite/tools/benchmark/BUILD b/tensorflow/lite/tools/benchmark/BUILD index 461acf0735d..0090705d44a 100644 --- a/tensorflow/lite/tools/benchmark/BUILD +++ b/tensorflow/lite/tools/benchmark/BUILD @@ -36,6 +36,26 @@ cc_binary( ], ) +cc_binary( + name = "benchmark_model_performance_options", + srcs = [ + "benchmark_tflite_performance_options_main.cc", + ], + copts = common_copts, + linkopts = tflite_linkopts() + select({ + "//tensorflow:android": [ + "-pie", # Android 5.0 and later supports only PIE + "-lm", # some builtin ops, e.g., tanh, need -lm + ], + "//conditions:default": [], + }), + deps = [ + ":benchmark_performance_options", + ":benchmark_tflite_model_lib", + ":logging", + ], +) + tf_cc_binary( name = "benchmark_model_plus_flex", srcs = [ @@ -99,6 +119,22 @@ cc_library( ], ) +cc_library( + name = "benchmark_performance_options", + srcs = [ + "benchmark_performance_options.cc", + ], + hdrs = ["benchmark_performance_options.h"], + copts = common_copts, + deps = [ + ":benchmark_model_lib", + ":benchmark_utils", + ":logging", + "//tensorflow/lite/profiling:time", + "//tensorflow/lite/tools:command_line_flags", + ], +) + cc_library( name = "benchmark_params", srcs = [ diff --git a/tensorflow/lite/tools/benchmark/README.md b/tensorflow/lite/tools/benchmark/README.md index 8e77a22f6b1..7b91d43eb82 100644 --- a/tensorflow/lite/tools/benchmark/README.md +++ b/tensorflow/lite/tools/benchmark/README.md @@ -213,3 +213,19 @@ Memory (bytes): count=0 Average inference timings in us: Warmup: 83235, Init: 38467, no stats: 79760.9 ``` + +## Benchmark multiple performance options in a single run + +A convenient and simple C++ binary is also provided to benchmark multiple +performance options in a single run. This binary is built based on the +aforementioned benchmark tool that could only benchmark a single performance +option at a time. They share the same build/install/run process, but the BUILD +target name of this binary is `benchmark_model_performance_options` and it takes +some additional parameters as detailed below. + +### Additional Parameters +* `perf_options_list`: `string` (default='all') \ + A comma-separated list of TFLite performance options to benchmark. +* `option_benchmark_run_delay`: `float` (default=-1.0) \ + The delay between two consecutive runs of benchmarking performance options + in seconds. diff --git a/tensorflow/lite/tools/benchmark/benchmark_model.cc b/tensorflow/lite/tools/benchmark/benchmark_model.cc index 488dc506dd3..1aee4caec37 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_model.cc +++ b/tensorflow/lite/tools/benchmark/benchmark_model.cc @@ -42,7 +42,7 @@ BenchmarkParams BenchmarkModel::DefaultParams() { BenchmarkModel::BenchmarkModel() : params_(DefaultParams()) {} -void BenchmarkLoggingListener::OnBenchmarkEnd(const BenchmarkResults &results) { +void BenchmarkLoggingListener::OnBenchmarkEnd(const BenchmarkResults& results) { auto inference_us = results.inference_time_us(); auto init_us = results.startup_latency_us(); auto warmup_us = results.warmup_time_us(); @@ -143,7 +143,7 @@ Stat BenchmarkModel::Run(int min_num_times, float min_secs, bool BenchmarkModel::ValidateParams() { return true; } -void BenchmarkModel::Run(int argc, char **argv) { +void BenchmarkModel::Run(int argc, char** argv) { if (!ParseFlags(argc, argv)) { return; } @@ -174,10 +174,10 @@ void BenchmarkModel::Run() { {startup_latency_us, input_bytes, warmup_time_us, inference_time_us}); } -bool BenchmarkModel::ParseFlags(int argc, char **argv) { +bool BenchmarkModel::ParseFlags(int* argc, char** argv) { auto flag_list = GetFlags(); const bool parse_result = - Flags::Parse(&argc, const_cast(argv), flag_list); + Flags::Parse(argc, const_cast(argv), flag_list); if (!parse_result) { std::string usage = Flags::Usage(argv[0], flag_list); TFLITE_LOG(ERROR) << usage; diff --git a/tensorflow/lite/tools/benchmark/benchmark_model.h b/tensorflow/lite/tools/benchmark/benchmark_model.h index 132ee84bb7b..0e783703396 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_model.h +++ b/tensorflow/lite/tools/benchmark/benchmark_model.h @@ -129,8 +129,9 @@ class BenchmarkLoggingListener : public BenchmarkListener { template Flag CreateFlag(const char* name, BenchmarkParams* params, const std::string& usage) { - return Flag(name, [params, name](const T& val) { params->Set(name, val); }, - params->Get(name), usage); + return Flag( + name, [params, name](const T& val) { params->Set(name, val); }, + params->Get(name), usage); } // Benchmarks a model. @@ -150,11 +151,19 @@ class BenchmarkModel { listeners_.AddListener(listener); } + BenchmarkParams* mutable_params() { return ¶ms_; } + + // Unparsable flags will remain in 'argv' in the original order and 'argc' + // will be updated accordingly. + bool ParseFlags(int* argc, char** argv); + protected: virtual void LogParams(); virtual bool ValidateParams(); - bool ParseFlags(int argc, char** argv); + + bool ParseFlags(int argc, char** argv) { return ParseFlags(&argc, argv); } virtual std::vector GetFlags(); + virtual uint64_t ComputeInputBytes() = 0; virtual tensorflow::Stat Run(int min_num_times, float min_secs, float max_secs, RunType run_type); diff --git a/tensorflow/lite/tools/benchmark/benchmark_performance_options.cc b/tensorflow/lite/tools/benchmark/benchmark_performance_options.cc new file mode 100644 index 00000000000..ea434341250 --- /dev/null +++ b/tensorflow/lite/tools/benchmark/benchmark_performance_options.cc @@ -0,0 +1,175 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/tools/benchmark/benchmark_performance_options.h" + +#include + +#include "tensorflow/lite/profiling/time.h" +#include "tensorflow/lite/tools/benchmark/benchmark_utils.h" +#include "tensorflow/lite/tools/benchmark/logging.h" +#include "tensorflow/lite/tools/command_line_flags.h" + +namespace tflite { +namespace benchmark { + +BenchmarkParams BenchmarkPerformanceOptions::DefaultParams() { + BenchmarkParams params; + params.AddParam("perf_options_list", + BenchmarkParam::Create("all")); + params.AddParam("option_benchmark_run_delay", + BenchmarkParam::Create(-1.0f)); + return params; +} + +std::vector BenchmarkPerformanceOptions::GetFlags() { + return { + CreateFlag( + "perf_options_list", ¶ms_, + "A comma-separated list of TFLite performance options to benchmark. " + "By default, all performance options are benchmarked."), + CreateFlag("option_benchmark_run_delay", ¶ms_, + "The delay between two consecutive runs of " + "benchmarking performance options in seconds."), + }; +} + +bool BenchmarkPerformanceOptions::ParseFlags(int* argc, char** argv) { + auto flag_list = GetFlags(); + const bool parse_result = + Flags::Parse(argc, const_cast(argv), flag_list); + if (!parse_result) { + std::string usage = Flags::Usage(argv[0], flag_list); + TFLITE_LOG(ERROR) << usage; + return false; + } + + // Parse the value of --perf_options_list to find performance options to be + // benchmarked. + return ParsePerfOptions(); +} + +bool BenchmarkPerformanceOptions::ParsePerfOptions() { + const auto& perf_options_list = params_.Get("perf_options_list"); + if (!util::SplitAndParse(perf_options_list, ',', &perf_options_)) { + TFLITE_LOG(ERROR) << "Cannot parse --perf_options_list: '" + << perf_options_list + << "'. Please double-check its value."; + perf_options_.clear(); + return false; + } + + const auto valid_options = GetValidPerfOptions(); + bool is_valid = true; + for (const auto& option : perf_options_) { + if (std::find(valid_options.begin(), valid_options.end(), option) == + valid_options.end()) { + is_valid = false; + break; + } + } + if (!is_valid) { + std::string valid_options_str; + for (int i = 0; i < valid_options.size() - 1; ++i) { + valid_options_str += (valid_options[i] + ", "); + } + valid_options_str += valid_options.back(); + TFLITE_LOG(ERROR) + << "There are invalid perf options in --perf_options_list: '" + << perf_options_list << "'. Valid perf options are: [" + << valid_options_str << "]"; + perf_options_.clear(); + return false; + } + return true; +} + +std::vector BenchmarkPerformanceOptions::GetValidPerfOptions() + const { + return {"all", "cpu", "gpu", "nnapi"}; +} + +bool BenchmarkPerformanceOptions::HasOption(const string& option) const { + return std::find(perf_options_.begin(), perf_options_.end(), option) != + perf_options_.end(); +} + +void BenchmarkPerformanceOptions::ResetPerformanceOptions() { + single_option_run_params_->Set("num_threads", 1); + single_option_run_params_->Set("use_gpu", false); + single_option_run_params_->Set("use_nnapi", false); +} + +void BenchmarkPerformanceOptions::BenchmarkCPUOptions() { + // Reset all performance-related options before any runs. + ResetPerformanceOptions(); + + const int num_threads[] = {1, 2, 4}; + for (int i = 0; i < sizeof(num_threads) / sizeof(int); ++i) { + single_option_run_params_->Set("num_threads", num_threads[i]); + util::SleepForSeconds(params_.Get("option_benchmark_run_delay")); + single_option_run_->Run(); + } +} + +void BenchmarkPerformanceOptions::BenchmarkGPUOptions() { + // Reset all performance-related options before any runs. + ResetPerformanceOptions(); + + single_option_run_params_->Set("use_gpu", true); + util::SleepForSeconds(params_.Get("option_benchmark_run_delay")); + single_option_run_->Run(); +} + +void BenchmarkPerformanceOptions::BenchmarkNnapiOptions() { + // Reset all performance-related options before any runs. + ResetPerformanceOptions(); + + single_option_run_params_->Set("use_nnapi", true); + util::SleepForSeconds(params_.Get("option_benchmark_run_delay")); + single_option_run_->Run(); +} + +void BenchmarkPerformanceOptions::Run(int argc, char** argv) { + // We first parse flags for single-option runs to get information like + // parameters of the input model etc. + if (!single_option_run_->ParseFlags(&argc, argv)) { + return; + } + + // Now, we parse flags that are specified for this particular binary. + if (!ParseFlags(&argc, argv)) { + return; + } + + TFLITE_LOG(INFO) << "The list of TFLite runtime options to be benchmarked: [" + << params_.Get("perf_options_list") << "]"; + + const bool benchmark_all = HasOption("all"); + if (benchmark_all || HasOption("cpu")) { + BenchmarkCPUOptions(); + } + + if (benchmark_all || HasOption("gpu")) { + BenchmarkGPUOptions(); + } + + if (benchmark_all || HasOption("nnapi")) { + BenchmarkNnapiOptions(); + } +} + +} // namespace benchmark +} // namespace tflite diff --git a/tensorflow/lite/tools/benchmark/benchmark_performance_options.h b/tensorflow/lite/tools/benchmark/benchmark_performance_options.h new file mode 100644 index 00000000000..a46d5bcf1ca --- /dev/null +++ b/tensorflow/lite/tools/benchmark/benchmark_performance_options.h @@ -0,0 +1,70 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_ +#define TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_ + +#include "tensorflow/lite/tools/benchmark/benchmark_model.h" + +namespace tflite { +namespace benchmark { + +// Benchmarks all performance options on a model by repeatedly invoking the +// single-performance-option run on a passed-in 'BenchmarkModel' object. +class BenchmarkPerformanceOptions { + public: + // Doesn't own the memory of 'single_option_run'. + explicit BenchmarkPerformanceOptions(BenchmarkModel* single_option_run) + : BenchmarkPerformanceOptions(DefaultParams(), single_option_run) {} + + BenchmarkPerformanceOptions(BenchmarkParams params, + BenchmarkModel* single_option_run) + : params_(std::move(params)), + single_option_run_(single_option_run), + single_option_run_params_(single_option_run->mutable_params()) {} + + virtual ~BenchmarkPerformanceOptions() {} + + virtual void Run(int argc, char** argv); + + protected: + static BenchmarkParams DefaultParams(); + + // Unparsable flags will remain in 'argv' in the original order and 'argc' + // will be updated accordingly. + bool ParseFlags(int* argc, char** argv); + virtual std::vector GetFlags(); + + bool ParsePerfOptions(); + virtual std::vector GetValidPerfOptions() const; + bool HasOption(const string& option) const; + virtual void ResetPerformanceOptions(); + + virtual void BenchmarkCPUOptions(); + virtual void BenchmarkGPUOptions(); + virtual void BenchmarkNnapiOptions(); + + BenchmarkParams params_; + std::vector perf_options_; + + // The object that drives a single-performance-option run. + BenchmarkModel* const single_option_run_; // Doesn't own the memory. + BenchmarkParams* const single_option_run_params_; // Doesn't own the memory. +}; + +} // namespace benchmark +} // namespace tflite + +#endif // TENSORFLOW_LITE_TOOLS_BENCHMARK_BENCHMARK_PERFORMANCE_OPTIONS_H_ diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_performance_options_main.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_performance_options_main.cc new file mode 100644 index 00000000000..c70a719423c --- /dev/null +++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_performance_options_main.cc @@ -0,0 +1,40 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/tools/benchmark/benchmark_performance_options.h" +#include "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h" +#include "tensorflow/lite/tools/benchmark/logging.h" + +namespace tflite { +namespace benchmark { + +int Main(int argc, char** argv) { +#ifdef TFLITE_CUSTOM_OPS_HEADER + TFLITE_LOG(INFO) << "STARTING with custom ops!"; +#else + TFLITE_LOG(INFO) << "STARTING!"; +#endif + BenchmarkTfLiteModel benchmark; + BenchmarkLoggingListener listener; + benchmark.AddListener(&listener); + + BenchmarkPerformanceOptions all_options_benchmark(&benchmark); + all_options_benchmark.Run(argc, argv); + return EXIT_SUCCESS; +} +} // namespace benchmark +} // namespace tflite + +int main(int argc, char** argv) { return tflite::benchmark::Main(argc, argv); }