Add an external delegate which is initialized from a shared library

This CL introduces the following API to manage an external delegate.
- TfLiteExternalDelegateCreate()
- TfLiteExternalDelegateOptionsDefault()
- TfLiteExternalDelegateDelete()
Also refactored the ExternalDelegateProvider to adopt this change.

PiperOrigin-RevId: 319171849
Change-Id: Ibccbb3a70aa91dfcf96745e13342cdbb2e77d832

tensorflow/lite/delegates/external/BUILD

@@ -0,0 +1,35 @@
+# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+package(
+    default_visibility = [
+        "//visibility:public",
+    ],
+    licenses = ["notice"],  # Apache 2.0
+)
+
+cc_library(
+    name = "external_delegate",
+    srcs = ["external_delegate.cc"],
+    hdrs = ["external_delegate.h"],
+    deps = [
+        "//tensorflow/lite:minimal_logging",
+        "//tensorflow/lite/c:common",
+    ],
+)
+
+exports_files([
+    "external_delegate.h",
+])

tensorflow/lite/delegates/external/external_delegate.cc

@@ -0,0 +1,243 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/lite/delegates/external/external_delegate.h"
+
+#include <string>
+#include <vector>
+
+#if defined(_WIN32)
+#include <Windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+#include "tensorflow/lite/minimal_logging.h"
+
+namespace tflite {
+namespace {
+
+// Library Support construct to handle dynamic library operations
+#if defined(_WIN32)
+struct LibSupport {
+  static void* Load(const char* lib) { return LoadLibrary(lib); }
+
+  static void* GetSymbol(void* handle, const char* symbol) {
+    return (void*)GetProcAddress((HMODULE)handle, symbol);
+  }
+
+  static int UnLoad(void* handle) { return FreeLibrary((HMODULE)handle); }
+};
+#else
+struct LibSupport {
+  static void* Load(const char* lib) {
+    return dlopen(lib, RTLD_LAZY | RTLD_LOCAL);
+  }
+
+  static void* GetSymbol(void* handle, const char* symbol) {
+    return dlsym(handle, symbol);
+  }
+
+  static int UnLoad(void* handle) { return dlclose(handle); }
+};
+#endif
+
+// External delegate library construct
+struct ExternalLib {
+  using CreateDelegatePtr = std::add_pointer<TfLiteDelegate*(
+      const char**, const char**, size_t,
+      void (*report_error)(const char*))>::type;
+  using DestroyDelegatePtr = std::add_pointer<void(TfLiteDelegate*)>::type;
+
+  // Open a given delegate library and load the create/destroy symbols
+  bool load(const std::string library) {
+    void* handle = LibSupport::Load(library.c_str());
+    if (handle == nullptr) {
+      TFLITE_LOG(TFLITE_LOG_INFO, "Unable to load external delegate from : %s",
+                 library.c_str());
+    } else {
+      create = reinterpret_cast<decltype(create)>(
+          LibSupport::GetSymbol(handle, "tflite_plugin_create_delegate"));
+      destroy = reinterpret_cast<decltype(destroy)>(
+          LibSupport::GetSymbol(handle, "tflite_plugin_destroy_delegate"));
+      return create && destroy;
+    }
+    return false;
+  }
+
+  CreateDelegatePtr create{nullptr};
+  DestroyDelegatePtr destroy{nullptr};
+};
+
+// An ExternalDelegateWrapper is responsibile to manage a TFLite delegate
+// initialized from a shared library. It creates a delegate from the given
+// option and storages it to external_delegate_ member variable. On the
+// destruction, it conducts necessary clean up process.
+class ExternalDelegateWrapper {
+ public:
+  explicit ExternalDelegateWrapper(
+      const TfLiteExternalDelegateOptions* options);
+  ~ExternalDelegateWrapper();
+
+  // Return a TfLiteDelegate which is created from
+  // tflite_plugin_create_delegate() of an external delegate logic.
+  TfLiteDelegate* tflite_external_delegate() { return external_delegate_; }
+
+  // Return a TfLiteDelegate which is convertibile to this class.
+  TfLiteDelegate* tflite_wrapper_delegate() { return &wrapper_delegate_; }
+
+ private:
+  ExternalLib external_lib_;
+
+  // external delegate instance owned by external delegate logic.
+  // It's created by "tflite_plugin_destroy_delegate()" function in the external
+  // delegate logic And it should be released by
+  // "tflite_plugin_destroy_delegate()" function.
+  TfLiteDelegate* external_delegate_;
+
+  // TfLiteDelegate representation of this ExternalDelegateWrapper object.
+  TfLiteDelegate wrapper_delegate_;
+};
+
+// Converts the given TfLiteDelegate to an ExternalDelegateWrapper instance.
+inline ExternalDelegateWrapper* GetExternalDelegateWrapper(
+    TfLiteDelegate* delegate) {
+  return reinterpret_cast<ExternalDelegateWrapper*>(delegate->data_);
+}
+
+// Relay Prepare() call to the associated external TfLiteDelegate object.
+TfLiteStatus DelegatePrepare(TfLiteContext* context, TfLiteDelegate* delegate) {
+  auto external_delegate_wrapper = GetExternalDelegateWrapper(delegate);
+  TfLiteDelegate* external_delegate =
+      external_delegate_wrapper->tflite_external_delegate();
+  return external_delegate->Prepare(context, external_delegate);
+}
+
+// Relay CopyFromBufferHandle() call to the associated external TfLiteDelegate
+// object.
+TfLiteStatus DelegateCopyFromBufferHandle(TfLiteContext* context,
+                                          struct TfLiteDelegate* delegate,
+                                          TfLiteBufferHandle buffer_handle,
+                                          TfLiteTensor* tensor) {
+  auto external_delegate_wrapper = GetExternalDelegateWrapper(delegate);
+  TfLiteDelegate* external_delegate =
+      external_delegate_wrapper->tflite_external_delegate();
+  return external_delegate->CopyFromBufferHandle(context, delegate,
+                                                 buffer_handle, tensor);
+}
+
+// Relay CopyToBufferHandle() call to the associated external TfLiteDelegate
+// object.
+TfLiteStatus DelegateCopyToBufferHandle(TfLiteContext* context,
+                                        struct TfLiteDelegate* delegate,
+                                        TfLiteBufferHandle buffer_handle,
+                                        TfLiteTensor* tensor) {
+  auto external_delegate_wrapper = GetExternalDelegateWrapper(delegate);
+  TfLiteDelegate* external_delegate =
+      external_delegate_wrapper->tflite_external_delegate();
+  return external_delegate->CopyToBufferHandle(context, delegate, buffer_handle,
+                                               tensor);
+}
+
+// Relay FreeBufferHandle() call to the associated external TfLiteDelegate
+// object.
+void DelegateFreeBufferHandle(TfLiteContext* context,
+                              struct TfLiteDelegate* delegate,
+                              TfLiteBufferHandle* handle) {
+  auto external_delegate_wrapper = GetExternalDelegateWrapper(delegate);
+  TfLiteDelegate* external_delegate =
+      external_delegate_wrapper->tflite_external_delegate();
+  return external_delegate->FreeBufferHandle(context, delegate, handle);
+}
+
+ExternalDelegateWrapper::ExternalDelegateWrapper(
+    const TfLiteExternalDelegateOptions* options) {
+  external_delegate_ = nullptr;
+  if (external_lib_.load(options->lib_path)) {
+    std::vector<const char*> ckeys, cvalues;
+    for (int i = 0; i < options->count; i++) {
+      ckeys.push_back(options->keys[i]);
+      cvalues.push_back(options->values[i]);
+    }
+
+    external_delegate_ = external_lib_.create(ckeys.data(), cvalues.data(),
+                                              ckeys.size(), nullptr);
+    if (external_delegate_) {
+      wrapper_delegate_ = {
+          .data_ = reinterpret_cast<void*>(this),
+          .Prepare = DelegatePrepare,
+          .CopyFromBufferHandle = nullptr,
+          .CopyToBufferHandle = nullptr,
+          .FreeBufferHandle = nullptr,
+          .flags = external_delegate_->flags,
+      };
+      if (external_delegate_->CopyFromBufferHandle) {
+        wrapper_delegate_.CopyFromBufferHandle = DelegateCopyFromBufferHandle;
+      }
+      if (external_delegate_->CopyToBufferHandle) {
+        wrapper_delegate_.CopyToBufferHandle = DelegateCopyToBufferHandle;
+      }
+      if (external_delegate_->FreeBufferHandle) {
+        wrapper_delegate_.FreeBufferHandle = DelegateFreeBufferHandle;
+      }
+    }
+  }
+}
+
+ExternalDelegateWrapper::~ExternalDelegateWrapper() {
+  if (external_delegate_ != nullptr) {
+    external_lib_.destroy(external_delegate_);
+  }
+}
+
+}  // namespace
+}  // namespace tflite
+
+// TfLiteExternalDelegateOptionsInsert adds key/value to the given
+// TfLiteExternalDelegateOptions instance.
+TfLiteStatus TfLiteExternalDelegateOptionsInsert(
+    TfLiteExternalDelegateOptions* options, const char* key,
+    const char* value) {
+  if (options->count >= kMaxOptions) {
+    return kTfLiteError;
+  }
+  options->keys[options->count] = key;
+  options->values[options->count] = value;
+  options->count++;
+  return kTfLiteOk;
+}
+
+TfLiteExternalDelegateOptions TfLiteExternalDelegateOptionsDefault(
+    const char* lib_path) {
+  TfLiteExternalDelegateOptions options = {
+      .lib_path = lib_path,
+      .count = 0,
+      .insert = TfLiteExternalDelegateOptionsInsert,
+  };
+  return options;
+}
+
+TfLiteDelegate* TfLiteExternalDelegateCreate(
+    const TfLiteExternalDelegateOptions* options) {
+  auto* external_delegate_wrapper =
+      new tflite::ExternalDelegateWrapper(options);
+  if (external_delegate_wrapper) {
+    return external_delegate_wrapper->tflite_wrapper_delegate();
+  }
+  return nullptr;
+}
+
+void TfLiteExternalDelegateDelete(TfLiteDelegate* delegate) {
+  delete tflite::GetExternalDelegateWrapper(delegate);
+}

tensorflow/lite/delegates/external/external_delegate.h

@@ -0,0 +1,53 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_LITE_DELEGATES_EXTERNAL_EXTERNAL_DELEGATE_H_
+#define TENSORFLOW_LITE_DELEGATES_EXTERNAL_EXTERNAL_DELEGATE_H_
+
+#include "tensorflow/lite/c/common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  // __cplusplus
+
+// TfLiteExternalDelegateOptions is a structure of key/value options to create
+// an external delegate.
+const int kMaxOptions = 256;
+typedef struct TfLiteExternalDelegateOptions {
+  const char* lib_path;
+  int count;
+  const char* keys[kMaxOptions];
+  const char* values[kMaxOptions];
+  TfLiteStatus (*insert)(TfLiteExternalDelegateOptions* options,
+                         const char* key, const char* value);
+} TfLiteExternalDelegateOptions;
+
+// Populates TfLiteExternalDelegateOptions with the given shared library path.
+TfLiteExternalDelegateOptions TfLiteExternalDelegateOptionsDefault(
+    const char* lib_path);
+
+// Creates a new delegate instance that need to be destroyed with
+// `TfLiteExternalDelegateDelete` when delegate is no longer used by TFLite.
+TfLiteDelegate* TfLiteExternalDelegateCreate(
+    const TfLiteExternalDelegateOptions* options);
+
+// Destroys a delegate created with `TfLiteExternalDelegateCreate` call.
+void TfLiteExternalDelegateDelete(TfLiteDelegate* delegate);
+
+#ifdef __cplusplus
+}
+#endif  // __cplusplus
+
+#endif  // TENSORFLOW_LITE_DELEGATES_EXTERNAL_EXTERNAL_DELEGATE_H_

tensorflow/lite/tools/delegates/BUILD

@@ -151,6 +151,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":delegate_provider_hdr",
+        "//tensorflow/lite/delegates/external:external_delegate",
     ],
     alwayslink = 1,
 )

tensorflow/lite/tools/delegates/external_delegate_provider.cc

@@ -12,45 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tensorflow/lite/tools/delegates/delegate_provider.h"
-
-#if defined(_WIN32)
-#include <Windows.h>
-#else
-#include <dlfcn.h>
-#endif
 
 #include <string>
-#include <type_traits>
 #include <vector>
 
+#include "tensorflow/lite/delegates/external/external_delegate.h"
+#include "tensorflow/lite/tools/delegates/delegate_provider.h"
+
 namespace tflite {
 namespace tools {
-namespace {
-// Library Support construct to handle dynamic library operations
-#if defined(_WIN32)
-struct LibSupport {
-  static void* Load(const char* lib) { return LoadLibrary(lib); }
-
-  static void* GetSymbol(void* handle, const char* symbol) {
-    return (void*)GetProcAddress((HMODULE)handle, symbol);
-  }
-
-  static int UnLoad(void* handle) { return FreeLibrary((HMODULE)handle); }
-};
-#else
-struct LibSupport {
-  static void* Load(const char* lib) {
-    return dlopen(lib, RTLD_LAZY | RTLD_LOCAL);
-  }
-
-  static void* GetSymbol(void* handle, const char* symbol) {
-    return dlsym(handle, symbol);
-  }
-
-  static int UnLoad(void* handle) { return dlclose(handle); }
-};
-#endif
 
 // Split a given string to a vector of string using a delimiter character
 std::vector<std::string> SplitString(const std::string& str, char delimiter) {
@@ -63,32 +33,6 @@ std::vector<std::string> SplitString(const std::string& str, char delimiter) {
   return tokens;
 }
 
-// External delegate library construct
-struct ExternalLib {
-  using CreateDelegatePtr = std::add_pointer<TfLiteDelegate*(
-      const char**, const char**, size_t,
-      void (*report_error)(const char*))>::type;
-  using DestroyDelegatePtr = std::add_pointer<void(TfLiteDelegate*)>::type;
-
-  // Open a given delegate library and load the create/destroy symbols
-  bool load(const std::string library) {
-    void* handle = LibSupport::Load(library.c_str());
-    if (handle == nullptr) {
-      TFLITE_LOG(INFO) << "Unable to load external delegate from : " << library;
-    } else {
-      create = reinterpret_cast<decltype(create)>(
-          LibSupport::GetSymbol(handle, "tflite_plugin_create_delegate"));
-      destroy = reinterpret_cast<decltype(destroy)>(
-          LibSupport::GetSymbol(handle, "tflite_plugin_destroy_delegate"));
-      return create && destroy;
-    }
-    return false;
-  }
-
-  CreateDelegatePtr create{nullptr};
-  DestroyDelegatePtr destroy{nullptr};
-};
-}  // namespace
 
 // External delegate provider used to dynamically load delegate libraries
 // Note: Assumes the lifetime of the provider exceeds the usage scope of
@@ -136,35 +80,28 @@ TfLiteDelegatePtr ExternalDelegateProvider::CreateTfLiteDelegate(
   TfLiteDelegatePtr delegate(nullptr, [](TfLiteDelegate*) {});
   std::string lib_path = params.Get<std::string>("external_delegate_path");
   if (!lib_path.empty()) {
-    ExternalLib delegate_lib;
-    if (delegate_lib.load(lib_path)) {
-      // Parse delegate options
-      const std::vector<std::string> options = SplitString(
-          params.Get<std::string>("external_delegate_options"), ';');
-      std::vector<std::string> keys, values;
-      for (const auto& option : options) {
-        auto key_value = SplitString(option, ':');
-        if (key_value.size() == 2) {
-          values.push_back(std::move(key_value[1]));
-          keys.push_back(std::move(key_value[0]));
-        }
-      }
+    auto delegate_options =
+        TfLiteExternalDelegateOptionsDefault(lib_path.c_str());
 
-      const size_t num_options = keys.size();
-      std::vector<const char*> ckeys, cvalues;
-      for (int i = 0; i < num_options; ++i) {
-        ckeys.push_back(keys[i].c_str());
-        cvalues.push_back(values[i].c_str());
+    // Parse delegate options
+    const std::vector<std::string> options =
+        SplitString(params.Get<std::string>("external_delegate_options"), ';');
+    std::vector<std::string> keys, values;
+    for (const auto& option : options) {
+      auto key_value = SplitString(option, ':');
+      if (key_value.size() == 2) {
+        delegate_options.insert(&delegate_options, key_value[0].c_str(),
+                                key_value[1].c_str());
       }
-
-      // Create delegate
-      delegate =
-          TfLiteDelegatePtr(delegate_lib.create(ckeys.data(), cvalues.data(),
-                                                num_options, nullptr),
-                            delegate_lib.destroy);
     }
+
+    auto external_delegate = TfLiteExternalDelegateCreate(&delegate_options);
+    return TfLiteDelegatePtr(external_delegate, [](TfLiteDelegate* delegate) {
+      TfLiteExternalDelegateDelete(delegate);
+    });
   }
   return delegate;
 }
+
 }  // namespace tools
 }  // namespace tflite