From e480d8f7ff66dbab239019c9f202748f6fa1f661 Mon Sep 17 00:00:00 2001 From: Chao Mei Date: Sun, 17 May 2020 21:07:20 -0700 Subject: [PATCH] Add a compile flag (i.e. adding "--define tflite_with_xnnpack=true" when using bazel to build, which corresponds to defining macro TFLITE_BUILD_WITH_XNNPACK_DELEGATE) to apply XNNPACK delegate in TFLite runtime. This is mainly to support Windows where weak symbols are not supported. PiperOrigin-RevId: 312011534 Change-Id: I27c6b206b8aa8ded2d2671c2fca843574f75752b --- tensorflow/lite/BUILD | 32 ++++++++++++ tensorflow/lite/interpreter_builder.cc | 26 +++------- .../lite/tflite_with_xnnpack_optional.cc | 52 +++++++++++++++++++ .../lite/tflite_with_xnnpack_optional.h | 26 ++++++++++ 4 files changed, 117 insertions(+), 19 deletions(-) create mode 100644 tensorflow/lite/tflite_with_xnnpack_optional.cc create mode 100644 tensorflow/lite/tflite_with_xnnpack_optional.h diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index 14babee2da7..ef25f03562f 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -246,6 +246,7 @@ cc_library( ":minimal_logging", ":simple_memory_arena", ":string", + ":tflite_with_xnnpack_optional", ":type_to_tflitetype", ":util", ":version", @@ -311,6 +312,8 @@ cc_library( ], ) +# Link this library to inject XNNPACK delegate to TFLite runtime automatically +# by utilizing the weak symbols if they're supported by the platform. cc_library( name = "tflite_with_xnnpack", srcs = ["tflite_with_xnnpack.cc"], @@ -323,6 +326,35 @@ cc_library( alwayslink = 1, ) +# Enables applying XNNPACK delegate for float models in TFLite runtime. +# WARNING: This build flag is experimental and subject to change. +config_setting( + name = "tflite_with_xnnpack_enabled", + values = {"define": "tflite_with_xnnpack=true"}, +) + +cc_library( + name = "tflite_with_xnnpack_optional", + srcs = ["tflite_with_xnnpack_optional.cc"], + hdrs = [ + "core/macros.h", + "tflite_with_xnnpack_optional.h", + ], + copts = tflite_copts() + TFLITE_DEFAULT_COPTS, + defines = select({ + ":tflite_with_xnnpack_enabled": ["TFLITE_BUILD_WITH_XNNPACK_DELEGATE"], + "//conditions:default": [], + }), + deps = [ + "//tensorflow/lite/c:common", + ] + select({ + ":tflite_with_xnnpack_enabled": [ + "//tensorflow/lite/delegates/xnnpack:xnnpack_delegate", + ], + "//conditions:default": [], + }), +) + cc_test( name = "string_util_test", size = "small", diff --git a/tensorflow/lite/interpreter_builder.cc b/tensorflow/lite/interpreter_builder.cc index fb87702fd13..43d81ef0770 100644 --- a/tensorflow/lite/interpreter_builder.cc +++ b/tensorflow/lite/interpreter_builder.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/lite/core/api/flatbuffer_conversions.h" #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/schema/schema_generated.h" +#include "tensorflow/lite/tflite_with_xnnpack_optional.h" #include "tensorflow/lite/util.h" #include "tensorflow/lite/version.h" @@ -108,27 +109,14 @@ TfLiteStatus ParseSparseIndexVector(const DimensionMetadata* src, const char* kEmptyTensorName = ""; -#if TFLITE_HAS_ATTRIBUTE_WEAK // Using weak symbols to create a delegate allows automatic injection of the // delegate simply by adding it as a dependency. - // For flex delegate, see also the strong override in // lite/delegates/flex/delegate.cc. TFLITE_ATTRIBUTE_WEAK Interpreter::TfLiteDelegatePtr AcquireFlexDelegate() { return Interpreter::TfLiteDelegatePtr(nullptr, [](TfLiteDelegate*) {}); } -// For XNNPACK delegate, see also the strong override in -// lite/tflite_with_xnnpack.cc. -TFLITE_ATTRIBUTE_WEAK Interpreter::TfLiteDelegatePtr AcquireXNNPACKDelegate( - int num_threads) { - return Interpreter::TfLiteDelegatePtr(nullptr, [](TfLiteDelegate*) {}); -} -#else -Interpreter::TfLiteDelegatePtr (*AcquireFlexDelegate)() = nullptr; -Interpreter::TfLiteDelegatePtr (*AcquireXNNPACKDelegate)(int) = nullptr; -#endif - namespace impl { InterpreterBuilder::InterpreterBuilder(const FlatBufferModel& model, @@ -541,17 +529,17 @@ TfLiteStatus InterpreterBuilder::ParseTensors( TfLiteStatus InterpreterBuilder::ApplyDelegates(Interpreter* interpreter, int num_threads) { // First, apply XNNPACK delegate if applicable. - if (AcquireXNNPACKDelegate && num_fp32_tensors_ > 0) { - if (auto xnnpack_delegate = AcquireXNNPACKDelegate(num_threads)) { - // The execution will fall back to default implementation if the XNNPACK - // delegate fails to be applied. Therefore, we ignore the return status - // here and let it fall through the rest of the code. + if (num_fp32_tensors_ > 0) { + // The execution will fall back to default implementation if the XNNPACK + // delegate fails to be applied. Therefore, we ignore the return status + // here and let it fall through the rest of the code. + if (auto xnnpack_delegate = MaybeCreateXNNPACKDelegate(num_threads)) { interpreter->ModifyGraphWithDelegate(std::move(xnnpack_delegate)); } } // Secondly, apply Flex delegate if applicable. - if (has_flex_op_ && AcquireFlexDelegate) { + if (has_flex_op_) { if (auto flex_delegate = AcquireFlexDelegate()) { return interpreter->ModifyGraphWithDelegate(std::move(flex_delegate)); } diff --git a/tensorflow/lite/tflite_with_xnnpack_optional.cc b/tensorflow/lite/tflite_with_xnnpack_optional.cc new file mode 100644 index 00000000000..31d4ff50f28 --- /dev/null +++ b/tensorflow/lite/tflite_with_xnnpack_optional.cc @@ -0,0 +1,52 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/tflite_with_xnnpack_optional.h" + +#include "tensorflow/lite/core/macros.h" + +#ifdef TFLITE_BUILD_WITH_XNNPACK_DELEGATE +#include "tensorflow/lite/delegates/xnnpack/xnnpack_delegate.h" +#endif + +namespace tflite { + +using TfLiteDelegatePtr = + std::unique_ptr; + +#ifndef TFLITE_BUILD_WITH_XNNPACK_DELEGATE +// Using weak symbols to create a delegate allows automatic injection of the +// delegate simply by adding it as a dependency. See the strong override in +// lite/tflite_with_xnnpack.cc, +TFLITE_ATTRIBUTE_WEAK TfLiteDelegatePtr +AcquireXNNPACKDelegate(int num_threads) { + return TfLiteDelegatePtr(nullptr, [](TfLiteDelegate*) {}); +} +#endif + +#ifdef TFLITE_BUILD_WITH_XNNPACK_DELEGATE +TfLiteDelegatePtr MaybeCreateXNNPACKDelegate(int num_threads) { + auto opts = TfLiteXNNPackDelegateOptionsDefault(); + // Note that we don't want to use the thread pool for num_threads == 1. + opts.num_threads = num_threads > 1 ? num_threads : 0; + return TfLiteDelegatePtr(TfLiteXNNPackDelegateCreate(&opts), + TfLiteXNNPackDelegateDelete); +} +#else +TfLiteDelegatePtr MaybeCreateXNNPACKDelegate(int num_threads) { + return AcquireXNNPACKDelegate(num_threads); +} +#endif + +} // namespace tflite diff --git a/tensorflow/lite/tflite_with_xnnpack_optional.h b/tensorflow/lite/tflite_with_xnnpack_optional.h new file mode 100644 index 00000000000..afbdbd17356 --- /dev/null +++ b/tensorflow/lite/tflite_with_xnnpack_optional.h @@ -0,0 +1,26 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_TFLITE_WITH_XNNPACK_OPTIONAL_H_ +#define TENSORFLOW_LITE_TFLITE_WITH_XNNPACK_OPTIONAL_H_ +#include + +#include "tensorflow/lite/c/common.h" + +namespace tflite { +std::unique_ptr +MaybeCreateXNNPACKDelegate(int num_threads); +} // namespace tflite + +#endif // TENSORFLOW_LITE_TFLITE_WITH_XNNPACK_OPTIONAL_H_