From a76ddd5b248fdcce7021e317b1018e4f9f8ad833 Mon Sep 17 00:00:00 2001 From: Fergus Henderson Date: Mon, 15 Feb 2021 03:17:09 -0800 Subject: [PATCH] Add C API for delegate plugins and in particular for the NNAPI delegate plugin. In particular: 1. Add C API types for delegate plugins. 2. Add a C API for the NNAPI Delegate Plugin. PiperOrigin-RevId: 357542308 Change-Id: I885e1cf81973613d066f5a0fe75b904c24b4ceb6 --- .../acceleration/configuration/BUILD | 13 ++ .../acceleration/configuration/c/BUILD | 44 +++++++ .../configuration/c/delegate_plugin.h | 60 +++++++++ .../configuration/c/nnapi_plugin.cc | 57 +++++++++ .../configuration/c/nnapi_plugin.h | 43 +++++++ .../configuration/nnapi_plugin.cc | 90 +------------ .../acceleration/configuration/nnapi_plugin.h | 120 ++++++++++++++++++ 7 files changed, 340 insertions(+), 87 deletions(-) create mode 100644 tensorflow/lite/experimental/acceleration/configuration/c/BUILD create mode 100644 tensorflow/lite/experimental/acceleration/configuration/c/delegate_plugin.h create mode 100644 tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.cc create mode 100644 tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.h create mode 100644 tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.h diff --git a/tensorflow/lite/experimental/acceleration/configuration/BUILD b/tensorflow/lite/experimental/acceleration/configuration/BUILD index ec8c66440c8..30ebd3e71eb 100644 --- a/tensorflow/lite/experimental/acceleration/configuration/BUILD +++ b/tensorflow/lite/experimental/acceleration/configuration/BUILD @@ -107,11 +107,24 @@ cc_library( cc_library( name = "nnapi_plugin", + deps = [ + ":nnapi_plugin_impl", + ], +) + +cc_library( + name = "nnapi_plugin_impl", srcs = ["nnapi_plugin.cc"], + hdrs = ["nnapi_plugin.h"], + visibility = [ + "//tensorflow/lite/experimental/acceleration/configuration/c:__pkg__", + ], deps = [ ":configuration_fbs", ":delegate_registry", + "//tensorflow/lite/c:common", "//tensorflow/lite/delegates/nnapi:nnapi_delegate", + "//tensorflow/lite/experimental/acceleration/configuration/c:delegate_plugin", "@com_google_absl//absl/memory", ], alwayslink = 1, # For registration to always run. diff --git a/tensorflow/lite/experimental/acceleration/configuration/c/BUILD b/tensorflow/lite/experimental/acceleration/configuration/c/BUILD new file mode 100644 index 00000000000..5762f0a5877 --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/configuration/c/BUILD @@ -0,0 +1,44 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# C API for delegate plugins. + +package( + default_visibility = ["//visibility:private"], + licenses = ["notice"], # Apache 2.0 +) + +cc_library( + name = "delegate_plugin", + hdrs = ["delegate_plugin.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/lite/c:common", + ], +) + +cc_library( + name = "nnapi_plugin", + srcs = ["nnapi_plugin.cc"], + hdrs = ["nnapi_plugin.h"], + visibility = ["//visibility:public"], + deps = [ + ":delegate_plugin", + "//tensorflow/lite/c:common", + "//tensorflow/lite/delegates/nnapi:nnapi_delegate", + "//tensorflow/lite/experimental/acceleration/configuration:configuration_fbs", + "//tensorflow/lite/experimental/acceleration/configuration:nnapi_plugin_impl", + ], +) diff --git a/tensorflow/lite/experimental/acceleration/configuration/c/delegate_plugin.h b/tensorflow/lite/experimental/acceleration/configuration/c/delegate_plugin.h new file mode 100644 index 00000000000..d5e6c3d4d6d --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/configuration/c/delegate_plugin.h @@ -0,0 +1,60 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_C_DELEGATE_PLUGIN_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_C_DELEGATE_PLUGIN_H_ + +// C API types for TF Lite delegate plugins. + +#include "tensorflow/lite/c/common.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Type of function to allocate and construct a delegate. +// The tflite_settings parameter should be a pointer to a FlatBuffer table +// object of type tflite::TFLiteSettings. (We use 'void *' here since this +// is a C API so we don't want to directly reference C++ types such +// as tflite::TFLiteSettings.) +typedef TfLiteDelegate *TfLiteDelegatePluginCreateFunc( + const void *tflite_settings); + +// Type of function to destroy and deallocate a delegate. +// The delegate argument must have been created with the corresponding +// create function from the same delegate plugin. +typedef void TfLiteDelegatePluginDestroyFunc(TfLiteDelegate *); + +// Type of function to return an error code for the last delegate operation. +// The delegate argument must have been created with the corresponding +// create function from the same delegate plugin. +typedef int TfLiteDelegatePluginGetDelegateErrnoFunc(TfLiteDelegate *); + +// Struct to hold all the methods for a delegate plugin. +typedef struct TfLiteDelegatePlugin { + // Function to allocate and construct a delegate. + TfLiteDelegatePluginCreateFunc *create; + + // Function to deallocate a delegate. + TfLiteDelegatePluginDestroyFunc *destroy; + + // Function to return an error code for the last delegate operation. + TfLiteDelegatePluginGetDelegateErrnoFunc *get_delegate_errno; +} TfLiteDelegatePlugin; + +#ifdef __cplusplus +}; // extern "C" +#endif + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_C_DELEGATE_PLUGIN_H_ diff --git a/tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.cc b/tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.cc new file mode 100644 index 00000000000..c9d94f5e610 --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.cc @@ -0,0 +1,57 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements the Delegate Plugin for the NNAPI Delegate. +// It provides both + +#include "tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.h" + +#include + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h" +#include "tensorflow/lite/experimental/acceleration/configuration/configuration_generated.h" +#include "tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.h" + +extern "C" { + +static TfLiteDelegate* CreateDelegate(const void* settings) { + const ::tflite::TFLiteSettings* tflite_settings = + static_cast(settings); + tflite::delegates::NnapiPlugin nnapi_plugin(*tflite_settings); + return new tflite::StatefulNnApiDelegate(nnapi_plugin.Options()); +} + +static void DestroyDelegate(TfLiteDelegate* delegate) { + delete static_cast(delegate); +} + +static int DelegateErrno(TfLiteDelegate* from_delegate) { + auto nnapi_delegate = + static_cast(from_delegate); + return nnapi_delegate->GetNnApiErrno(); +} + +static constexpr TfLiteDelegatePlugin kPluginCApi{ + CreateDelegate, + DestroyDelegate, + DelegateErrno, +}; + +const TfLiteDelegatePlugin* TfLiteNnapiDelegatePluginCApi() { + return &kPluginCApi; +} + +} // extern "C" diff --git a/tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.h b/tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.h new file mode 100644 index 00000000000..cef0b441fc3 --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/configuration/c/nnapi_plugin.h @@ -0,0 +1,43 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_C_NNAPI_PLUGIN_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_C_NNAPI_PLUGIN_H_ + +// This header file is for the delegate plugin for NNAPI. +// +// For the C++ delegate plugin interface, the NNAPI delegate plugin is added to +// the DelegatePluginRegistry by the side effect of a constructor for a static +// object, so there's no public API needed for this plugin, other than the API +// of tflite::delegates::DelegatePluginRegistry, which is declared in +// delegate_registry.h. +// +// But to provide a C API to access the NNAPI delegate plugin, we do expose +// some functions, which are declared below. + +#include "tensorflow/lite/experimental/acceleration/configuration/c/delegate_plugin.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// C API for the NNAPI delegate plugin. +// Returns a pointer to a statically allocated table of function pointers. +const TfLiteDelegatePlugin* TfLiteNnapiDelegatePluginCApi(); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_C_NNAPI_PLUGIN_H_ diff --git a/tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.cc b/tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.cc index 33b481dffab..fdda69a72cf 100644 --- a/tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.cc +++ b/tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.cc @@ -12,98 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include -#include "absl/memory/memory.h" -#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h" -#include "tensorflow/lite/experimental/acceleration/configuration/configuration_generated.h" -#include "tensorflow/lite/experimental/acceleration/configuration/delegate_registry.h" +// This file implements the TFLite Delegate Plugin for the NNAPI Delegate. + +#include "tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.h" namespace tflite { namespace delegates { -inline tflite::StatefulNnApiDelegate::Options::ExecutionPreference -ConvertExecutionPrefence( - NNAPIExecutionPreference from_compatibility_preference) { - using TflitePreference = - tflite::StatefulNnApiDelegate::Options::ExecutionPreference; - switch (from_compatibility_preference) { - case NNAPIExecutionPreference_NNAPI_LOW_POWER: - return TflitePreference::kLowPower; - case NNAPIExecutionPreference_NNAPI_FAST_SINGLE_ANSWER: - return TflitePreference::kFastSingleAnswer; - case NNAPIExecutionPreference_NNAPI_SUSTAINED_SPEED: - return TflitePreference::kSustainedSpeed; - default: - return TflitePreference::kUndefined; - } -} - -inline int ConvertExecutionPriority( - NNAPIExecutionPriority from_compatibility_priority) { - switch (from_compatibility_priority) { - case NNAPIExecutionPriority_NNAPI_PRIORITY_LOW: - return ANEURALNETWORKS_PRIORITY_LOW; - case NNAPIExecutionPriority_NNAPI_PRIORITY_MEDIUM: - return ANEURALNETWORKS_PRIORITY_MEDIUM; - case NNAPIExecutionPriority_NNAPI_PRIORITY_HIGH: - return ANEURALNETWORKS_PRIORITY_HIGH; - default: - return ANEURALNETWORKS_PRIORITY_DEFAULT; - } -} - -class NnapiPlugin : public DelegatePluginInterface { - public: - TfLiteDelegatePtr Create() override { - auto nnapi_delegate = - absl::make_unique(options_); - return TfLiteDelegatePtr( - nnapi_delegate.release(), [](TfLiteDelegate* delegate) { - delete static_cast(delegate); - }); - } - int GetDelegateErrno(TfLiteDelegate* from_delegate) override { - auto nnapi_delegate = - static_cast(from_delegate); - return nnapi_delegate->GetNnApiErrno(); - } - static std::unique_ptr New( - const TFLiteSettings& tflite_settings) { - return absl::make_unique(tflite_settings); - } - explicit NnapiPlugin(const TFLiteSettings& tflite_settings) { - const NNAPISettings* nnapi_settings = tflite_settings.nnapi_settings(); - if (!nnapi_settings) return; - if (nnapi_settings->accelerator_name() && - nnapi_settings->accelerator_name()->Length() != 0) { - accelerator_ = nnapi_settings->accelerator_name()->str(); - options_.accelerator_name = accelerator_.c_str(); - } - if (nnapi_settings->cache_directory() && - nnapi_settings->cache_directory()->Length() != 0) { - cache_dir_ = nnapi_settings->cache_directory()->str(); - options_.cache_dir = cache_dir_.c_str(); - } - if (nnapi_settings->model_token() && - nnapi_settings->model_token()->Length() != 0) { - model_token_ = nnapi_settings->model_token()->str(); - options_.model_token = model_token_.c_str(); - } - options_.execution_preference = - ConvertExecutionPrefence(nnapi_settings->execution_preference()); - options_.disallow_nnapi_cpu = - !nnapi_settings->allow_nnapi_cpu_on_android_10_plus(); - options_.execution_priority = - ConvertExecutionPriority(nnapi_settings->execution_priority()); - options_.allow_fp16 = nnapi_settings->allow_fp16_precision_for_fp32(); - } - - private: - std::string accelerator_, cache_dir_, model_token_; - tflite::StatefulNnApiDelegate::Options options_; -}; - TFLITE_REGISTER_DELEGATE_FACTORY_FUNCTION(NnapiPlugin, NnapiPlugin::New); } // namespace delegates diff --git a/tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.h b/tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.h new file mode 100644 index 00000000000..bf70a0e5bbe --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/configuration/nnapi_plugin.h @@ -0,0 +1,120 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_NNAPI_PLUGIN_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_NNAPI_PLUGIN_H_ + +// This file provides the NNApiPlugin class, which implements the +// TFLite Delegate Plugin for the NNAPI Delegate. + +#include +#include + +#include "absl/memory/memory.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h" +#include "tensorflow/lite/experimental/acceleration/configuration/c/delegate_plugin.h" +#include "tensorflow/lite/experimental/acceleration/configuration/configuration_generated.h" +#include "tensorflow/lite/experimental/acceleration/configuration/delegate_registry.h" + +namespace tflite { +namespace delegates { + +class NnapiPlugin : public DelegatePluginInterface { + public: + TfLiteDelegatePtr Create() override { + auto nnapi_delegate = + absl::make_unique(options_); + return TfLiteDelegatePtr( + nnapi_delegate.release(), [](TfLiteDelegate* delegate) { + delete static_cast(delegate); + }); + } + int GetDelegateErrno(TfLiteDelegate* from_delegate) override { + auto nnapi_delegate = + static_cast(from_delegate); + return nnapi_delegate->GetNnApiErrno(); + } + static std::unique_ptr New( + const TFLiteSettings& tflite_settings) { + return absl::make_unique(tflite_settings); + } + explicit NnapiPlugin(const TFLiteSettings& tflite_settings) { + const NNAPISettings* nnapi_settings = tflite_settings.nnapi_settings(); + if (!nnapi_settings) return; + if (nnapi_settings->accelerator_name() && + nnapi_settings->accelerator_name()->Length() != 0) { + accelerator_ = nnapi_settings->accelerator_name()->str(); + options_.accelerator_name = accelerator_.c_str(); + } + if (nnapi_settings->cache_directory() && + nnapi_settings->cache_directory()->Length() != 0) { + cache_dir_ = nnapi_settings->cache_directory()->str(); + options_.cache_dir = cache_dir_.c_str(); + } + if (nnapi_settings->model_token() && + nnapi_settings->model_token()->Length() != 0) { + model_token_ = nnapi_settings->model_token()->str(); + options_.model_token = model_token_.c_str(); + } + options_.execution_preference = + ConvertExecutionPrefence(nnapi_settings->execution_preference()); + options_.disallow_nnapi_cpu = + !nnapi_settings->allow_nnapi_cpu_on_android_10_plus(); + options_.execution_priority = + ConvertExecutionPriority(nnapi_settings->execution_priority()); + options_.allow_fp16 = nnapi_settings->allow_fp16_precision_for_fp32(); + } + const tflite::StatefulNnApiDelegate::Options& Options() { return options_; } + + private: + static inline tflite::StatefulNnApiDelegate::Options::ExecutionPreference + ConvertExecutionPrefence( + NNAPIExecutionPreference from_compatibility_preference) { + using TflitePreference = + tflite::StatefulNnApiDelegate::Options::ExecutionPreference; + switch (from_compatibility_preference) { + case NNAPIExecutionPreference_NNAPI_LOW_POWER: + return TflitePreference::kLowPower; + case NNAPIExecutionPreference_NNAPI_FAST_SINGLE_ANSWER: + return TflitePreference::kFastSingleAnswer; + case NNAPIExecutionPreference_NNAPI_SUSTAINED_SPEED: + return TflitePreference::kSustainedSpeed; + default: + return TflitePreference::kUndefined; + } + } + + static inline int ConvertExecutionPriority( + NNAPIExecutionPriority from_compatibility_priority) { + switch (from_compatibility_priority) { + case NNAPIExecutionPriority_NNAPI_PRIORITY_LOW: + return ANEURALNETWORKS_PRIORITY_LOW; + case NNAPIExecutionPriority_NNAPI_PRIORITY_MEDIUM: + return ANEURALNETWORKS_PRIORITY_MEDIUM; + case NNAPIExecutionPriority_NNAPI_PRIORITY_HIGH: + return ANEURALNETWORKS_PRIORITY_HIGH; + default: + return ANEURALNETWORKS_PRIORITY_DEFAULT; + } + } + + std::string accelerator_, cache_dir_, model_token_; + tflite::StatefulNnApiDelegate::Options options_; +}; + +} // namespace delegates +} // namespace tflite + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_CONFIGURATION_NNAPI_PLUGIN_H_