From 19c8b341121be77ff56faf8fb28ea705dc92837a Mon Sep 17 00:00:00 2001 From: Sachin Joglekar Date: Tue, 20 Oct 2020 16:59:31 -0700 Subject: [PATCH] Update detection intro page with better details on support & tooling. Also add an 'implementing delegates' page that will be polished later with simpler instructions on delegate authoring. PiperOrigin-RevId: 338165590 Change-Id: Ifd6e0fd261e157c41321f7dacc2fb16f40bd7769 --- tensorflow/lite/g3doc/_book.yaml | 4 +- .../lite/g3doc/performance/delegates.md | 430 +++++++++--------- .../performance/images/delegate_runtime.png | Bin 0 -> 35166 bytes .../performance/implementing_delegate.md | 171 +++++++ 4 files changed, 393 insertions(+), 212 deletions(-) create mode 100644 tensorflow/lite/g3doc/performance/images/delegate_runtime.png create mode 100644 tensorflow/lite/g3doc/performance/implementing_delegate.md diff --git a/tensorflow/lite/g3doc/_book.yaml b/tensorflow/lite/g3doc/_book.yaml index 22fd564635c..37a3682f473 100644 --- a/tensorflow/lite/g3doc/_book.yaml +++ b/tensorflow/lite/g3doc/_book.yaml @@ -139,7 +139,6 @@ upper_tabs: path: /lite/performance/measurement - title: "Delegates" path: /lite/performance/delegates - status: experimental - title: "GPU delegate" path: /lite/performance/gpu - title: "Advanced GPU" @@ -152,6 +151,9 @@ upper_tabs: - title: "Core ML delegate" path: /lite/performance/coreml_delegate status: experimental + - title: "Implementing a delegate" + path: /lite/performance/implementing_delegate + status: experimental - heading: "Optimize a model" - title: "Overview" diff --git a/tensorflow/lite/g3doc/performance/delegates.md b/tensorflow/lite/g3doc/performance/delegates.md index 6b233075398..b17c9c35fec 100644 --- a/tensorflow/lite/g3doc/performance/delegates.md +++ b/tensorflow/lite/g3doc/performance/delegates.md @@ -1,31 +1,61 @@ -# TensorFlow Lite delegates +# TensorFlow Lite Delegates -Note: Delegate API is still experimental and is subject to change. +## Introduction -## What is a TensorFlow Lite delegate? +**Delegates** enable hardware acceleration of TensorFlow Lite models by +leveraging on-device accelerators such as the GPU and +[Digital Signal Processor (DSP)](https://en.wikipedia.org/wiki/Digital_signal_processor). -A TensorFlow Lite delegate is a way to delegate part or all of graph execution -to another executor. +By default, TensorFlow Lite utilizes CPU kernels that are optimized for the +[ARM Neon](https://developer.arm.com/documentation/dht0002/a/Introducing-NEON/NEON-architecture-overview/NEON-instructions) +instruction set. However, the CPU is a multi-purpose processor that isn't +necessarily optimized for the heavy arithmetic typically found in Machine +Learning models (for example, the matrix math involved in convolution and dense +layers). -## Why should I use delegates? +On the other hand, most modern mobile phones contain chips that are better at +handling these heavy operations. Utilizing them for neural network operations +provides huge benefits in terms of latency and power efficiency. For example, +GPUs can provide upto a +[5x speedup](https://blog.tensorflow.org/2020/08/faster-mobile-gpu-inference-with-opencl.html) +in latency, while the +[Qualcomm® Hexagon DSP](https://developer.qualcomm.com/software/hexagon-dsp-sdk/dsp-processor) +has shown to reduce power consumption upto 75% in our experiments. -Running inference on compute-heavy machine learning models on mobile devices is -resource demanding due to the devices' limited processing and power. +Each of these accelerators have associated APIs that enable custom computations, +such as [OpenCL](https://www.khronos.org/opencl/) or +[OpenGL ES](https://www.khronos.org/opengles/) for mobile GPU and the +[Qualcomm® Hexagon SDK](https://developer.qualcomm.com/software/hexagon-dsp-sdk) +for DSP. Typically, you would have to write a lot of custom code to run a neural +network though these interfaces. Things get even complicated when you consider +that each accelerator has its pros & cons and cannot execute every operation in +a neural network. TensorFlow Lite's Delegate API solves this problem by acting +as a bridge between the TFLite runtime and these lower-level APIs. -Instead of relying on the CPU, some devices have hardware accelerators, such as -GPU or DSP, that allows for better performance and higher energy efficiency. +![runtime with delegates](images/delegate_runtime.png) -## Using the built-in delegates +## Choosing a Delegate -TensorFlow Lite provides the following delegates for hardware acceleration: +TensorFlow Lite supports multiple delegates, each of which is optimized for +certain platform(s) and particular types of models. Usually, there will be +multiple delegates applicable to your use-case, depending on two major criteria: +the *Platform* (Android or iOS?) you target, and the *Model-type* +(floating-point or quantized?) that you are trying to accelerate. + +### Delegates by Platform + +#### Cross-platform (Android & iOS) + +* **GPU delegate** - The GPU delegate can be used on both Android and iOS. It + is optimized to run 32-bit and 16-bit float based models where a GPU is + available. It also supports 8-bit quantized models and provides GPU + performance on par with their float versions. For details on the GPU + delegate, see [TensorFlow Lite on GPU](gpu_advanced.md). For step-by-step + tutorials on using the GPU delegate with Android and iOS, see + [TensorFlow Lite GPU Delegate Tutorial](gpu.md). + +#### Android -* **GPU delegate for cross platform acceleration** - The GPU delegate can be - used on both Android and iOS. It is optimized to run 32-bit and 16-bit float - based models where a GPU is available. It also supports 8-bit quantized - models and provides GPU performance on par with their float versions. For - details on the GPU delegate, see [TensorFlow Lite on GPU](gpu_advanced.md). - For step-by-step tutorials on using the GPU delegate with Android and iOS, - see [TensorFlow Lite GPU Delegate Tutorial](gpu.md). * **NNAPI delegate for newer Android devices** - The NNAPI delegate can be used to accelerate models on Android devices with GPU, DSP and / or NPU available. It is available in Android 8.1 (API 27+) or higher. For an @@ -33,210 +63,188 @@ TensorFlow Lite provides the following delegates for hardware acceleration: practices, see [TensorFlow Lite NNAPI delegate](nnapi.md). * **Hexagon delegate for older Android devices** - The Hexagon delegate can be used to accelerate models on Android devices with Qualcomm Hexagon DSP. It - can be used on devices older version of Android OS that does not fully - support NNAPI. See [TensorFlow Lite Hexagon delegate](hexagon_delegate.md) - for more detail. + can be used on devices running older versions of Android that do not support + NNAPI. See [TensorFlow Lite Hexagon delegate](hexagon_delegate.md) for more + detail. + +#### iOS + * **Core ML delegate for newer iPhones and iPads** - For newer iPhones and iPads where Neural Engine is available, you can use Core ML delegate to - accelerate inference for 32-bit float based models. Neural Engine is - available Apple mobile devices with A12 SoC or higher. For an overview of - the Core ML delegate and step-by-step instructions, see + accelerate inference for 32-bit or 16-bit floating-point models. Neural + Engine is available Apple mobile devices with A12 SoC or higher. For an + overview of the Core ML delegate and step-by-step instructions, see [TensorFlow Lite Core ML delegate](coreml_delegate.md). -## How do delegates work? +### Delegates by model type -Let's say we have a simple model graph such as the following: +Each accelerator is designed with a certain bit-width of data in mind. If you +provide a floating-point model to a delegate that only supports 8-bit quantized +operations (such as the [Hexagon delegate](hexagon_delegate.md)), it will reject +all its operations and the model will run entirely on the CPU. To avoid such +surprises, the table below provides an overview of delegate support based on +model type: -![Original graph](../images/performance/tflite_delegate_graph_1.png "Original Graph") +**Model Type** | **GPU** | **NNAPI** | **Hexagon** | **CoreML** +------------------------------------------------------------------------------------------------------- | ------- | --------- | ----------- | ---------- +Floating-point (32 bit) | Yes | Yes | No | Yes +[Post-training float16 quantization](post_training_float16_quant.ipynb) | Yes | No | No | Yes +[Post-training dynamic range quantization](post_training_quant.ipynb) | Yes | Yes | No | No +[Post-training integer quantization](post_training_integer_quant.ipynb) | Yes | Yes | Yes | No +[Quantization-aware training](http://www.tensorflow.org/model_optimization/guide/quantization/training) | Yes | Yes | Yes | No -If a delegate was provided for specific operations, then TensorFlow Lite will -split the graph into multiple subgraphs where each subgraph will be handled by a -delegate. +### Validating performance -Let's assume that a delegate, `MyDelegate`, has a faster implementation for -Conv2D and Mean operations. The resulting main graph will be updated to look -like below. +The information in this section acts as a rough guideline for shortlisting the +delegates that could improve your application. However, it is important to note +that each delegate has a pre-defined set of operations it supports, and may +perform differently depending on the model and device; for example, the +[NNAPI delegate](nnapi.md) may choose to use Google's Edge-TPU on a Pixel phone +while utilizing a DSP on another device. Therefore, it is usually recommended +that you perform some benchmarking to gauge how useful a delegate is for your +needs. This also helps justify the binary size increase associated with +attaching a delegate to the TensorFlow Lite runtime. -![Graph with delegate](../images/performance/tflite_delegate_graph_2.png "Graph with delegate") +TensorFlow Lite has extensive performance and accuracy-evaluation tooling that +can empower developers to be confident in using delegates in their application. +These tools are discussed in the next section. -Each subgraph that is handled by a delegate will be replaced with a node that -evaluates the subgraph on its invoked call. +## Tools for Evaluation -Depending on the model, the final graph can end up with one node, which means -that all of the graphs were delegated or multiple nodes handled the subgraphs. -In general, you don't want to have multiple subgraphs handled by the delegate, -since each time you switch from delegate to the main graph, there is an overhead -for passing the results from the subgraph to the main graph. It's not always -safe to share memory. +### Latency & memory footprint -## How to add a delegate +TensorFlow Lite’s +[benchmark tool](https://www.tensorflow.org/lite/performance/measurement) can be +used with suitable parameters to estimate model performance, including average +inference latency, initialization overhead, memory footprint, etc. This tool +supports multiple flags to figure out the best delegate configuration for your +model. For instance, `--gpu_backend=gl` can be specified with `--use_gpu` to +measure GPU execution with OpenGL. The complete list of supported delegate +parameters is defined in the +[detailed documentation](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/delegates/README.md#tflite-delegate-registrar). -_Note that the API used below is experimental and is subject to change._ +Here’s an example run for a quantized model with GPU via `adb`: -Based on the previous section, to add a delegate, we need to do the following: - -1. Define a kernel node that is responsible for evaluating the delegate - subgraph. -1. Create an instance of - [TfLiteDelegate](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/common.h#L611), - which is responsible for registering the kernel node and claiming the nodes - that the delegate can execute. - -To see it in code, let's define a delegate and call it `MyDelegate`, which can -execute Conv2D and Mean operations faster. - -```c++ -#include "tensorflow/lite/util.h" -#include "tensorflow/lite/builtin_ops.h" -#include "tensorflow/lite/context_util.h" - -// This is where the execution of the operations or whole graph happens. -// The class below has an empty implementation just as a guideline -// on the structure. -class MyDelegate { - public: - // Returns true if my delegate can handle this type of op. - static bool SupportedOp(const TfLiteRegistration* registration) { - switch (registration->builtin_code) { - case kTfLiteBuiltinConv2d: - case kTfLiteBuiltinMean: - return true; - default: - return false; - } - } - - // Any initialization code needed - bool Init() {} - // Any preparation work needed (e.g. allocate buffers) - bool Prepare(TfLiteContext* context, TfLiteNode* node) {} - // Actual running of the delegate subgraph. - bool Invoke(TfLiteContext* context, TfLiteNode* node) {} - // ... Add any other methods needed. -}; - -// Create the TfLiteRegistration for the Kernel node which will replace -// the subgraph in the main TfLite graph. -TfLiteRegistration GetMyDelegateNodeRegistration() { - // This is the registration for the Delegate Node that gets added to - // the TFLite graph instead of the subgraph it replaces. - // It is treated as an OP node. But in our case - // Init will initialize the delegate. - // Invoke will run the delegate graph. - // Prepare for preparing the delegate. - // Free for any cleaning needed by the delegate. - TfLiteRegistration kernel_registration; - kernel_registration.builtin_code = kTfLiteBuiltinDelegate; - kernel_registration.custom_name = "MyDelegate"; - kernel_registration.free = [](TfLiteContext* context, void* buffer) -> void { - delete reinterpret_cast(buffer); - }; - kernel_registration.init = [](TfLiteContext* context, const char* buffer, - size_t) -> void* { - // In the node init phase, initialize MyDelegate instance - const TfLiteDelegateParams* delegate_params = - reinterpret_cast(buffer); - MyDelegate* my_delegate = new MyDelegate; - if (!my_delegate->Init(context, params)) { - return nullptr; - } - return my_delegate; - }; - kernel_registration.invoke = [](TfLiteContext* context, - TfLiteNode* node) -> TfLiteStatus { - MyDelegate* kernel = reinterpret_cast(node->user_data); - return kernel->Invoke(context, node); - }; - kernel_registration.prepare = [](TfLiteContext* context, - TfLiteNode* node) -> TfLiteStatus { - MyDelegate* kernel = reinterpret_cast(node->user_data); - return kernel->Prepare(context, node); - }; - - return kernel_registration; -} - -// TfLiteDelegate methods - -TfLiteStatus DelegatePrepare(TfLiteContext* context, TfLiteDelegate* delegate) { - // Claim all nodes that can be evaluated by the delegate and ask the - // framework to update the graph with delegate kernel instead. - std::vector supported_nodes; - TfLiteIntArray* plan; - TF_LITE_ENSURE_STATUS(context->GetExecutionPlan(context, &plan)); - TfLiteNode* node; - TfLiteRegistration* registration; - for (int node_index : TfLiteIntArrayView(plan)) { - TF_LITE_ENSURE_STATUS(context->GetNodeAndRegistration( - context, node_index, &node, ®istration)); - if (MyDelegate::SupportedOp(registration)) { - supported_nodes.push_back(node_index); - } - } - TfLiteRegistration my_delegate_kernel_registration = - GetMyDelegateNodeRegistration(); - - // This call split the graphs into subgraphs, for subgraphs that can be - // handled by the delegate, it will replace it with a - // 'my_delegate_kernel_registration' - TfLiteIntArray* supported_nodes_int_array = - ::tflite::ConvertVectorToTfLiteIntArray(supported_nodes); - auto status = context->ReplaceNodeSubsetsWithDelegateKernels( - context, my_delegate_kernel_registration, - supported_nodes_int_array, delegate); - TfLiteIntArrayFree(supported_nodes_int_array); - return status -} - -void FreeBufferHandle(TfLiteContext* context, TfLiteDelegate* delegate, - TfLiteBufferHandle* handle) { - // Do any cleanups. -} - -TfLiteStatus CopyToBufferHandle(TfLiteContext* context, - TfLiteDelegate* delegate, - TfLiteBufferHandle buffer_handle, - TfLiteTensor* tensor) { - // Copies data from tensor to delegate buffer if needed. - return kTfLiteOk; -} - -TfLiteStatus CopyFromBufferHandle(TfLiteContext* context, - TfLiteDelegate* delegate, - TfLiteBufferHandle buffer_handle, - TfLiteTensor* tensor) { - // Copies the data from delegate buffer into the tensor raw memory. - return kTfLiteOk; -} - -// Caller takes ownership of the returned pointer. -TfLiteDelegate* CreateMyDelegate() { - TfLiteDelegate* delegate = new TfLiteDelegate; - - delegate->data_ = nullptr; - delegate->flags = kTfLiteDelegateFlagsNone; - delegate->Prepare = &DelegatePrepare; - // This cannot be null. - delegate->CopyFromBufferHandle = &CopyFromBufferHandle; - // This can be null. - delegate->CopyToBufferHandle = &CopyToBufferHandle; - // This can be null. - delegate->FreeBufferHandle = &FreeBufferHandle; - - return delegate; -} - - -// To add the delegate you need to call - -auto* my_delegate = CreateMyDelegate(); -if (interpreter->ModifyGraphWithDelegate(my_delegate) != - kTfLiteOk) { - // Handle error -} else { - interpreter->Invoke(); -} -... -// Don't forget to delete your delegate -delete my_delegate; ``` +adb shell /data/local/tmp/benchmark_model \ + --graph=/data/local/tmp/mobilenet_v1_224_quant.tflite \ + --use_gpu=true +``` + +You can download pre-built version of this tool for Android, 64-bit ARM +architecture +[here](https://storage.googleapis.com/tensorflow-nightly-public/prod/tensorflow/release/lite/tools/nightly/latest/android_aarch64_benchmark_model.apk) +([more details](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/benchmark/android)). + +### Accuracy & correctness + +Delegates usually perform computations at a different precision than their CPU +counterparts. As a result, there is an (usually minor) accuracy tradeoff +associated with utilizing a delegate for hardware acceleration. Note that this +isn't *always* true; for example, since the GPU uses floating-point precision to +run quantized models, there might be a slight precision improvement (for e.g., +<1% Top-5 improvement in ILSVRC image classification). + +TensorFlow Lite has two types of tooling to measure how accurately a delegate +behaves for a given model: *Task-Based* and *Task-Agnostic*. All the tools +described in this section support the +[advanced delegation parameters](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/delegates/README.md#tflite-delegate-registrar) +used by the benchmarking tool from the previous section. Note that the +sub-sections below focus on *delegate evaluation* (Does the delegate perform the +same as the CPU?) rather than model evaluation (Is the model itself good for the +task?). + +#### Task-Based Evaluation + +TensorFlow Lite has tools to evaluate correctness on two image-based tasks: + +* [ILSVRC 2012](http://image-net.org/challenges/LSVRC/2012/) (Image + Classification) with + [top-K accuracy](https://en.wikipedia.org/wiki/Evaluation_measures_\(information_retrieval\)#Precision_at_K) + +* [COCO Object Detection (w/ bounding boxes)](https://cocodataset.org/#detection-2020) + with + [mean Average Precision (mAP)](https://en.wikipedia.org/wiki/Evaluation_measures_\(information_retrieval\)#Mean_average_precision) + +Prebuilt binaries of these tools (Android, 64-bit ARM architecture), along with +documentation can be found here: + +* [ImageNet Image Classification](https://storage.googleapis.com/tensorflow-nightly-public/prod/tensorflow/release/lite/tools/nightly/latest/android_aarch64_eval_imagenet_image_classification) + ([More details](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification)) +* [COCO Object Detection](https://storage.googleapis.com/tensorflow-nightly-public/prod/tensorflow/release/lite/tools/nightly/latest/android_aarch64_eval_coco_object_detection) + ([More details](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/coco_object_detection)) + +The example below demonstrates +[image classification evaluation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/imagenet_image_classification) +with NNAPI utilizing Google's Edge-TPU on a Pixel 4: + +``` +adb shell /data/local/tmp/run_eval \ + --model_file=/data/local/tmp/mobilenet_quant_v1_224.tflite \ + --ground_truth_images_path=/data/local/tmp/ilsvrc_images \ + --ground_truth_labels=/data/local/tmp/ilsvrc_validation_labels.txt \ + --model_output_labels=/data/local/tmp/model_output_labels.txt \ + --output_file_path=/data/local/tmp/accuracy_output.txt \ + --num_images=0 # Run on all images. \ + --use_nnapi=true \ + --nnapi_accelerator_name=google-edgetpu +``` + +The expected output is a list of Top-K metrics from 1 to 10: + +``` +Top-1 Accuracy: 0.733333 +Top-2 Accuracy: 0.826667 +Top-3 Accuracy: 0.856667 +Top-4 Accuracy: 0.87 +Top-5 Accuracy: 0.89 +Top-6 Accuracy: 0.903333 +Top-7 Accuracy: 0.906667 +Top-8 Accuracy: 0.913333 +Top-9 Accuracy: 0.92 +Top-10 Accuracy: 0.923333 +``` + +#### Task-Agnostic Evaluation + +For tasks where there isn't an established on-device evaluation tool, or if you +are experimenting with custom models, TensorFlow Lite has the +[Inference Diff](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/tools/evaluation/tasks/inference_diff) +tool. (Android, 64-bit ARM binary architecture binary +[here](https://storage.googleapis.com/tensorflow-nightly-public/prod/tensorflow/release/lite/tools/nightly/latest/android_aarch64_eval_inference_diff)) + +Inference Diff compares TensorFlow Lite execution (in terms of latency & +output-value deviation) in two settings: + +* Single-threaded CPU Inference +* User-defined Inference - defined by + [these parameters](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/delegates/README.md#tflite-delegate-registrar) + +To do so, the tool generates random Gaussian data and passes it through two +TFLite Interpreters - one running single-threaded CPU kernels, and the other +parametrized by the user's arguments. + +It measures the latency of both, as well as the absolute difference between the +output tensors from each Interpreter, on a per-element basis. + +For a model with a single output tensor, the output might look like this: + +``` +Num evaluation runs: 50 +Reference run latency: avg=84364.2(us), std_dev=12525(us) +Test run latency: avg=7281.64(us), std_dev=2089(us) +OutputDiff[0]: avg_error=1.96277e-05, std_dev=6.95767e-06 +``` + +What this means is that for the output tensor at index `0`, the elements from +the CPU output different from the delegate output by an average of `1.96e-05`. + +Note that interpreting these numbers requires deeper knowledge of the model, and +what each output tensor signifies. If its a simple regression that determines +some sort of score or embedding, the difference should be low (otherwise it's an +error with the delegate). However, outputs like the 'detection class' one from +SSD models is a little harder to interpret. For example, it might show a +difference using this tool, but that may not mean something really wrong with +the delegate: consider two (fake) classes: "TV (ID: 10)", "Monitor (ID:20)" - If +a delegate is slightly off the golden truth and shows monitor instead of TV, the +output diff for this tensor might be something as high as 20-10 = 10. diff --git a/tensorflow/lite/g3doc/performance/images/delegate_runtime.png b/tensorflow/lite/g3doc/performance/images/delegate_runtime.png new file mode 100644 index 0000000000000000000000000000000000000000..e229f0fda092ceca75d0587fc0e7e51cf8e12209 GIT binary patch literal 35166 zcmeAS@N?(olHy`uVBq!ia0y~yU=CtnVC3OoVqjo!j2HOFz`(#*9OUlAu@6OCLHr}6j{qUJN#wKjdj58)M2sto$FmNg` zsx+`zSuw8bvvZsyk0dsM!I9yt-5*PYstE_=9U0Ec`WXxv%dNvRZHSUAE zfWzkZ{Sj$%RFI6j@aFJ*kX0~+e*{@By!ov>O;Wv+1tzAzX!83*I@C>I!Fm=(li&Y3 zQjd8mAv8D4)@@p~YSpY+Qrki!ZoK38aAo;4p55!#ty{Eck?JIk&+I}Eo;`c^^yyz_ z^}b*?kgG~0m~zhS>+0%SvSbOLvGW`be*XSVn~J($lxQ2&Mk>Ve0DY`t#?H z?yTxNuf;kS3J7KBHp+#q&HbqR>EXop`6OqVQ!j$E))qt5>a3s&|UA`kQ=BcdCn}%7r4P_WD=WpAsj&UvFWT z_47r55a%B?mSw&Y_kuR}OKqGoMP%x6A8XYK)23a!HSzPukDVPITegXwD$ij$dh}>$ zXsB#;pM|wb1Iq-5cJ&55S=qYI!X~6BP~cc_v)!H@N#utz$AX*R-F1xpgdkB24$h2k z56>gV`#%n*jBkGhPxnlA0RS6tm*05C!V~09s6vplmMnE9?5Isi2Su(|FP}ASvZ~OdMf!8k`D~QGHRpi8_Vh_2P zKhOG^`+8%jPt@(BS0`JS<;2{ISzV~icE@V>?~iBypPg;C&2r|psr;KaSG+7q`+se! z@9gulY+fzBEcvSc^Hpc*uyZeUbUi)gZf^ay^zzLA*L0V7%`n!wyK7Qz>F;l8dikMp z@4s#366aEe=A{ho#yqZAmloEiR^G11>vFt*-(P%oh2pi;r=t3I?oT)@C@eT_l1O-9 zSjO!ab)v%3+=p9@-`?NrU-48nA|T|>wu9!^PaQkN!LY^j>JpWs#YNn+y6%?M1%`*S zGYHK+qT~DZ%T`mdbvGwYoayW-^*XzggIjd^lpxiVW9M#7S-OOGeg;ZVU$A1T?u*;= z_knAOPtEIDf7324Zi`5NE*6j$Uz2p|&(zzWxVvtdorynss=RzG?{4`los(zv zr@T?QcK^Sps_d=r5`N^I+zYF~q3L54!9quhL z)!gXsnaMdnD0gkoN+F}<1&b9G^xMRnOnmHU+k|KlMWOG!E21KV=~CEYLSMw|(aPXzT6%!;9}R9}xMy z&{(~fg@Lv5=5)Vwh68zbnr_|N)qcId;uqU{`#M$zJ=xn*^C#3izPFjx_;~h;((`Ba z_f-DB865sz?iAPO<^93ouj3*^|DMu*aAoOkQ&Urh181Jx4=rB%OVWT}lp(H~J~kLakRiCpl3?Rde3-@E0G|eyn=@_*0jcSDbfj>8&4$dqbvXzs~yGtn3%R%Uf1^rlxbQ<{i+STk|w`zG4RfYm3kT0jB#@FoNtG(E(`y{%hB)e6Esx;8QP;(`72l%5+&1ms`S6|Z>1De2PV8}?I%k&Pr*C%;{97iTXAvGQ z+$O^C#oYYK^XJKyN-C?>`Xo)#U(LAkWsUl}xC0G!$E>W)bS&%d{ycVf`I3ePhWiia zX1^%;6&$WFCzrRnj_vpMb$4o?9P&QjaG_(yt<_r}yYXr0>efGd#>-Gznv*1U>f|Yz zJNs_gTRgg_yuD6s*J?4}$gY$oo629$Pmg0P z`~3tj!zJU*j1N|)?AWoR zEH6e&P3w)k(8+`iGc6cal)PTGMx*}6mC{2AUpb>}jvu^s>BXC?zppHrmG+`UZ?4_m zNx{Y*v-ufzyok}a{qx*UHak7H=;JZwO`QuA_IED)7976*)-Ufy->+Ntu$~QNr*?dr;d1r$+x~Apa>(WT(sYJMyBpWu1Q%KDY(A=UGuSor z+mwrP`wm+3&s8st+wn2s@F~m9YP~#s-Ce!ESs3OnRc$`B=>OfyIeTi#zJ8nBd^=1x z-aysBL`zG}PD`yWq5krlJ595{$IdV{etxw|L)NPD?MH5Zh8<78{5du=_Ocq!{@)kg zJ~?`u!C~=5zjyZyau&D-Z$59XaK~I>PKc_uvSs3z11o0y$lfl`-0+tpn%8uy?{5D9 zreC*SRh#K=m(MJJdrsKV@n2~7a@D2F*5BE&f5nV~<{dS!A5JUf?)|z&X!7M5$JrR3 z=Yfo@Sr@2(hjecHtCCs4gKWf7z9`UM&WI;XB(`f=%OdrGvnajjk4yyxp{jvjTf zoMm5)tVa3Xo zmL;FEe0|>rSaWZ(Y(8jF@=5DyK|oAU)tfuIj*NBO!M4-tX8%5OR$pbC(CgsL$e+J| zsIWB`zC9+_#~wv{_8E9?L5@?Fcei7~L<{?^^qSug$7 zF~VcQAK}l6hKib3Px;6*oZ;Ep+REY|v%~jR4MRX|Z1ImLTc7o;UG{fd3lGB_i}xoE zK3jY-Pg>}hpMSIMU3$UacbAv1JMvDmxa#An z@G!R^@4f7i8R@B$K4^YDd|V}M=C-GW+t=sb*&qsShcxQ5T+6o=l92lPbhq8}iadR9 zPKFO33Qm_zt9p5LgQYvC%kg8;Mv>t>Y|TF}Y~HYfg@J|TheoB*;RFM@9ed{4*Um`_ z-??PU7e(e@Q^P}cRhh6vB#MQFP3rge(9_gS*3;3_`};$4>F%KLO$OHL_R8ARBOl~` zX}@*LO0`w*OsmW%&6Sd8zJ+;aMg|sY7V8KVG5o37b7uL~GhF{?nFlVNedyI$_M*=n zFVt;p?!=}nefl&-F2?$d4jaRhyLLy{74fh=&b`9^?ADt*f41;E`m#-RrD}= z277iaIdksRliVxpk&>Qg@7emr{wk`S9CWpfXI5=)Lz^$S`Rl>JSyRr`J1-*j%JOMn zKAt&SeeB4@!|YLWb{3t#cIn31!|T_qS(NE4)q5;wf+6Pxh8yu=$&)fOw{1JwYQJrr z=-HLhMhosI+^$pCzn5tCIK$*t_WJ|v=R2E|pM^y47x!3f`skR>g+n#oCu&M9b96cH zNAKPDs;c(t%->rLuG&_8V=MhE#IWv8RY+*8tjxc62d+(J{ylr%u9Ei1d1u1IukWa4 zzZD@NXIos{Fx4|w_VLo?^}!p>rYyL5oh{=A3#UTkHwTA{3_Qym%Zn$x;`qu&dp~c@`O=IpL7BHry1h@uRsG=KVASdDAtoxARDFn>LEe5T z&juqwArTE-?d4WNf?`2ouO5Bdb#TdqX|GhYl~3+@xP0loX20iNUVh7G{n_E;FS>O9 zGry|0!X@7qYG3s(E%mKR+cqUdecJ7=JvKH^5=(--ta{yEFTSVzB$wx^SJb9p@yM4? zw)m`SabBjHVqv1bT}9WYM00<#*NrP1-f!D4F8%uJ$4|+&2l@MZYQNkmzQYl%zh7Y6 z#M*iLKgk-NxOBy%x?yTzLPpB$X62w}MdOO?{j3~yN{lKO{&8}296qyqgAv1-=?yjU z4}=-@@&u-}$%Prz@aK53bnmZwrILGH*i62e(LDX{JG1-#Pd|OO+Fkp*Ea+1C4)L#h z-QN8A)tfgz?u`2`HY=%T$Ed!J=;)hTT)RqM_` ztNqDtE)f~0cd=$l-u&Gkn-=){%vtx%<^2cE(n5CbnRBMsc(+=Q6bFOdzr>}fPrK86 z1q8*ESaOO6)8(6s zK3&tBd+_*aXVZm7hVnv_f-XJxl~&eQKm6#p^T{oW&AVZl<3}&&iB*U199LM$b5&H} z6-VQAMeS)fUJ1YF>S{|cn}6fKbTLba&Vy4bEB>A{{B^V7rL|bwJC#%ObDJJW=df&e zaa`EL%d4oYEG#tC)X>n-%uGyN{Qt+=qE`=}?M+wvRr@d2_WUjTFUd)(nrhO$ga5v` z(Vu(o!|xe0garROddF_vxz>)Mqt|;+!QIMlQ!Z-V71{IGnQ_7ViG7or?zFx>x3RGE zNkQ>v?x#;v&dJ_;^HBQCS>Nr3=98uf+^&71RJmDzK`Hssah|vT-@mFg@TQ-wNA>h;W#-09yvkY^XLR?Di@DXe#Z#8-=-nAu z>DubP*Dxw7tnR^&slBV8&9mS1b82`(YUtF~PCa{h`On4aQ=GQVoHilx_5`ghTMbsN zSn-9wbBWgppVHE^vlkS22ipoJ9(**(ZM)XV-JR@x@ppHp{#QbTe_tD z?WLI|KT=mNvdh0yEjE8fl6?_FX4KJIAFJZ0a-!=b86--3QtRK(G_SqW;W(|t>Tm6x z<#*lnTHITG_i9=VL{%r%#nhX)3(-0lBB^tk(#?(K5>4zjwc1|R2_%}S4pjuw}Ys{HloZf?DP z-p>{LyxN+z1QQ*VZGGoA6m0X;lCwK#CgwJ6*{>_pv^KAL@Zm*z-o51Z+Z8`sUG{Y+ zuVZ2`=ihG`?6&_C*Md3gBUx^JIQgt8Ju))V%Zp22fBlpxA<`yElkayqL`ey5U6Fo% zo~~qQUaindf8B}hGws_Q3Tt28+PZqdf`slg+3!+)wLhosb@koXePN6A0&BrfLW%q3 zraU!W9hA$UQ@|)C9wWh=KX;aBcBt#+-L-Q?5?k#%P916wFUS(OVz%edO|7;29e$Ya zzg@mV_Z<80@9+6J6ZgibDkvo6MwPdWSG2*Zyj{MS}Hq}zCuDEqYsA3pHg!o}Ps`9hFMjgrB8!^F?2E{9t; zzR9(lc<#ts=e2<@?xs^G{1$Y|J2bmEAo9;r@z1IT-|AbIpP1WvWJOO}V@kP(egkii z!g9{nN($<7+rBHf{q+oW2cDw;ae~T`{9MBAh9(O65mzfne~9XGpA|V#%*70yd)epy}A%89yH@yi;Eh$Y$5x zhP_{pt(sL4&zWY?G)HVo%E6fnTc%z!(VTrKC;iYx7KT+`js6lMydCyw+z;MxJhD^r zp7oaHm2>1NDZAggNJags@``4GtWB+~T?>lksL@ zmXWdW(YlA5b&vd$VMzSkdxPCpCvlsw%!PKXW!w*D9d?Xp6ZmH5*t5zwIKjZM@{qIY z6E&L{_STgL^xfVh-O5$bG2ZJbP_lMmaqq)~`X+hQ3-T3~- zboPBwHw2FUa{!HrHgJ@u2*lhY5E`MrcN(&d2t4rDvzT6}vROHa*`h~^$htijxpTV}d zzuWId4MRb-o#3_!K3lH}JoLG-@3qo_0;@f@+I3ZPkwtit_{1t=6ua_VI7?Tu~9x`8x`JA7Op8^j5ylyL*ji-tXBO zd3zgk`t3Vn(%ZzD?*Dh+x%Z6B@lKWr4(`ehY_Ff!_`Y3T+*kAA=(EKwhu|M3W>tUvF zqSvFNW?Hu=bgu0$^De!6dD@1&&Gt1Lf39G&{3!6T;m4)SJIa~?b3EU0Gc{cO)OI;> zlg-8V;Wuj*<$B$@v-ZYg0daA8n~D$D)<&1Vzjrq_^l`SCJnsYP{}X&Z&aanh%>T3T z;gj{>qpabz9I<(@MX@ zSZ$k4sSL*n?LQAzSpJ^fz4q$=?k7{HO!3I}cUyK(;J@o(hK9(=?!I#7M~>U)Tz>aU zdxnp4;m+DeKeg{WFSpzM%pmmdg*OM^u3oBgM10-7gMS*{^S}LQtu=j;i2c6j^IzQf z@&DW0e|=fjZCZP7UpKranBK@TfkEIN=fiSNfy;RUe8Mu5vg(xtWIra{N(2stnbJmMJE&tzq z_g$ITVt4hQOU3kMow|@$|X{hfSG2b)t~auNwaaYZptJ9=UaH*S(|r)=EWP>RT#q zD43|}Sty#?I&I6}Oruj>pDxR`o%ZnlvqkP^*_XZlpC8&LtOK6M`6I)${h5$s|I(@h zlddZ)KDN*QZl}TVnKxd2-hbfP#F({#7rHC%?+v^p<;s%6sBkYLJ9n4m?1HMdcck0u z?W*)nXMb2EQQXqeduY;|{=@c$N0!|x_c1)bP-AX~pv4ROu(f-djsz=p#|vJVrIum!}=E zdYYn-;%+?(KN6=fpTotjW%rwTZ`SB3FVcu!uf+5~&_BadpgPL<)#98AnMC17PgVzX z8y_sxQ%v30($VX)j6J*KL-pO?2ZQ(@&0oS_^gPbsu3*peh8Z>5hc48VGPqosDs{-1 zxuN1@@+H?p+nF0~1iwjSJzcZFTR!RaMuVxn%O3e8c?T7f6icAQaR(Y-J6-Wn^?d8{ypNBL?q9o`O#dyWod-&fZSJ(!S-ERY$DB{Q?9Om` zi!unVxLGCIbI0GsZ=Ft-!9Ru$cTKr#x2=3<*KAdJ#Q1U7tdBV}Dg?h}2(DlBv|ef6 zg&CF8cj$hfE-tuz={mNGRu13Pp9LrXy}rn}|CVOA!x833Qzve8KUl!_arq{e9hK7G zq%PkRN?bbg{IZqXEDH-R&Z{VN$u~+g5nm=~cjC!urOfn2ZR4g*zs|ZH zv69i#TQ_05;k~+r!Ef4kRsBeRzTtk_;{k*NDk7IYw zmo0DH`2yaq^H_Y*?uh!`$``N1zf60RbF@h_&Fz-)^t^fYwQFWhL4%>eu~U?}H*k&u*KJG+}a2;82uHM8TySiehTJgoKhRD_i607G8gEr>m#Q z%D`C|SmeRldNlY=yM|`TNo|dqCy%Dd?lXt21t>iI($Y90D%WZ&x$X%{2o&f;%4$8Arb+CrZ<-=>NSoPWHp+U>}?>~pcxH|+Y9u-@$Bh1PAh zB|%M#cszJn=La8o_h7rq@~17+H>Q*yI{f%jg;Q{o{+pN1F0L=F#g^<6-gy6#a%bG0 ze)E?2pVQvm*!I=dH|CsiJX6*-g>?*1TU*?{FO+iZxbebOVA{68{)Fvn5$jd%J$bvt zW5)9Ne>!*9-c!F3-KBH(g%JU+9&F198lRjUcn|~*_B5>C1 z+13BAO%1(zm4DvecS_p1cm3xHEYyU>5Y?YFP42 zd5MX6;G@*rzuNWY{kuJf<9E??-3OamYegj+Q==aT)G!>`ExyjW!(L|6b;A!<$M)IT z3QaQ?-emg9@b{`S6^d1^#4iiO+}}7vHm^F+dsctM@MU;(dG5)Rz?Qp-w}Af)270R<<`RIg;~-;f&1?H&Wl`BSa-DP$-mX*^J0G8 zuTO5fWn%h!)@&Vy8~q!u1_a;S^1CJ9q51LjecOJ`JA3-A`>ojBpO0L8xwF*1`QNj) zJh^`p-@lSiR#{Z@zo|t}c0Yqv8En~1gD#`{^5-vS-fYQdpLk&6~aV#{rd59-|hdAMrk|F<~|oM zo&WxAY48iD#TP+ysf9eNr*Dr*PihW}^t7{6+__}S#J!;*F~2?rdzCM3YhNzO;}+yq z`{3iRe_dVP3=R$ru@UlF4qs<3nNo5pLS^1IO|`tEAGrnk&)zAkTX)er_3WgV-08R4 z-S>X45`OehI3_gqZiM6F^*>skxZRhrpCa-k!z8NWNwt{fr!+b5#LNDZq9a&dU(1Y0 zdGjN%?5@b}3C)Qg<2AMPo<4o}@X?~H*UY+V?%heLtk|*pgxBfQCvA3=%}C{Wa@TI! zq4*!2Y4`P8`eR`$HWE}nsC`&lUvT8g$HQ?|Yyr{bOD^nO{eAI$^$jK(T1sooL30-~ z4NqQRkjfEvt4&&!mAUQP=dF8|@t!)ReJ#iChIi=Ah&V9EuPwZ`JX}r8p;n(dmPfnXQ^>W6R>-QS@=T`>>|FDT% z)-D^G(U`%-xID(^Wl&y`2?WbN(aD^eCu_KlO1 zE6RTRc=;5@Fg2$prZexeIyksBwMz@D-`=kYzrV1pzu)|sv(uUO7H@O;tn8I<|K~3~ z{K9d$-rdZFuPzlGQjYKmnGv~f_9B~~KQ(9G$au5*rm=<1tjLDcWoJUq{5CX9A?B=JBd;bJ6ig zzPo$+x9?Y9a5DJE?yaocFPsDz(yZ6ZEt-iKc`s$wPt3It*`1fzK@+|Ht&Yqi6EPTGr zoZX+6f9UwkEG2#Q;>Ty!aW0lRw14vQz41#OP5(Pe-g*1mF(FVFnr|An3GXQT{*pmE zIdhr2EZ@96WhLKt{eS!O_FZiSUFF^Fo0@iZ=DC!}yj&w7J-y=LuFr2Y7l-cBtoYr| z}5|d`~TXfFV815EL|x#_uSVzcc0yQXKMbn)U(U0nU<2cN95ukl&6YF(VglEoBb@f+2j0~xE)o$w{RZrE!pbg z+PZ1eXTAGX6{lZT`sN;fyHWc4yrb&NZuH9-WObM_903oT$DOWCTiF*iwesP=Wh+*s zJi2l4u{+NL_e*Ext#OvB;h(p6 zU)^`kmoqNinQ&uT@r)c_-?OjIvdg;hZ`o>c?d9dwD^^^&tNx+u@!~mm+AcKgtYp3Y zeg0hg;7;#x@&+Y3|IH$pu*EoQhBtj0G z*E8#B`Ef!c2ehuxmvg~QVil_A2dhOiK^HSVST6WuWBBj-TP7P8mpL+&|Gst9=%E5^ zsh0wy%7?X}MX6y;H{_74dkb3r_%>-~!2}QJ+CNa&GmI(YTdlTZPCwLa22O>>GSKqa zvW-3#9-skMkW)c3&f1J7zxPd5*alnb1zN?qTcIH>es_eL4Du4$eN!1`)!z3KxD6}q zK_L(;;IMgptd1}r@;c7@UJPgV<*RnQMRNNEJ&px8uj@_YWQP^UASdOkHl*F(E_CD@ zlG`)HSuVWE7C+t63?40jg!h5%LJphXM{_1sBe{JGH&e#9t=*}I9FdmaK8WUAaP#|G zmW}WVSfzpG7AvC(w&2{cKa>&Ta}(7Ihq?HSXNe_q3T4EzEVJgB{mx)bZLg$zH;aY9 zfxC8m1sk}Kf?c+qVOFhdx^5wSh2;bWp$85OXNSAO^5ZkP&VlCp@3%YL$u9W${<|b! z-;&1y5s&^EKQH=u?eT{dH|F0|{?^K}kGpZ7R2FF7C$u#!9+J=xd{+?o{;N8}QP$&u zxG+b5&HrxqhP%p**MwiJUU+PqpxmcTN1IxaQp1Z3mJ4rQXJ~hQc7(1)1g-szX3F@s zaa19L5Gld4qyA8fU7c6ycfPK!pEKso42}%d)zi|s9(l^c|MDfDa37Z+>O2g)fB15^ z?CZY2;d9HHo+mGv|Nl{QK9mT~g}3w@6U&}yi}`W(c0}%ddBK17(WS}Jn~OhR&-qtr z`)|QYL;oiazJ&$}J>m|3ucNCSzcJy{1jfH?J+VKUfBoH>RlnwDcvM*3qeHRz{|%mb zK6@tq?3uXRu_O0Os_ZJ?EJ!SSc4q4gcc!y~9F@|-(%(JTwol5NA9wcTmc^f@Y|+u* zf5GpOQuD9b+ZHE222Gp&sO7j3A;H+NZP~0TODg>G4!$?uVAg*mVt;>D`O($e<+n#V zOG{nKFiCZ-RCHuiRBXJo>62MThP!Xwf}7tpRg-@=7DrnC++%a+Z%^r(GbOuEm5Paq zYU$~9|D5pR(&{{A#3b9<(6gmwT2(*$`M7?Gl+5J37cc%S zVm@=uB|iH^L!JH2H^~MPpTqKmg+xxAI`^7^$I}0y`^;L^rBAK-V^5qq`F@$g_Oz2% z*2P){Kc4VHB`fR3@hcnpSGWJaseLYLZ*@TE3`CxYayoGAjsQd6z4E}ounikcUP+|I zd#n9ZPT5raX|v*g_l$pcOa5tWe*fd=ZOO=;n~Ww|o9T#&2_;y+Wt`@AtZJ6hA*Y;d zcq4;BD5I~TcWL00-8Sa-|1)mL1AwuO_*{_Cns%5uKl5~rL&!Ub8~ZI9$mfd zzbNUECgZuSW!H`+x82k@b$Z&=@D{ftal3vRnA&z{ot|^?#ZsfvKNs%VPR%sA_4|jv z`MaGhE&G*O^&l&A|DRm#y7}1ZvhbQehd#fUIa}$F(`DJV@5a~Osp!p`AOGa3eAUNe z&6_rHoxNw<_a?^Ktmb8j6vMR~OP^IiK@(;t*@r}X`+79ko7%p<=^g)k`Mc86Qig=P z=Mx^UvMipKFPo8Tv|<0s6&)95yj=b2m4vYxpSk%pK|w*U^Cv1Fy(<0xQJby#piR!S zKiBq)YigIx+*P3LZOKqFajM;>?A+MgyYcn!&z?E=^y$NyCV3NjTPIFgQV|!o@c8Tr ztNZ#-YirFm`(x`}*Ku?Dj5+V5e@u(mtQtIf^G^?%oZNjaOSUY}vzz$-QSm3P7M34R zcb~X1Lpox{i=yJonjQWzd(X!dN+o1&NjX|N8<-4Z;E18<&o(vvVF(C~{d`LM!d-0*Q{~;e zmbrO3-Er9ceFEd(lqFVyxhw0$&&j<0@@4Dp@_?$ecPkD~yR60&yYpp0jT4vV&nsu2 zubj=#An`Ku-{u;%t2P#Y?h1#`KeA-Yqe++i9xd7%vYS^j;o53xzt$(**wS-dUd+UzYc8@A36c_m7E*iF%g^U4FCHC}~0Kq$U~T&zjukqSL3u zS!`OlV8xGTf5k6ul=nV;>bbA9ZHdGFX@3-A_xwEZ^JR8WqEpbV6DwCUIJiW-xOH-E zi2nRZm*sahGOf2M;R(f_~M%tC>Zi@qr=CnI@CbDrVH1@eWER7CNJ*wC4 zAIsiw-OjQ5xzC2JCU(_d?MyGf3xd5Gq*iY=}^<(v%EZ|rqkBlpFL%g-%EAlw6XE-^Y^mRrmG=S zj`6%*zi`>6lE3fHthrMhKKD$|mg<#d*7av48Gi)%g> z&0X3*clS}FAJb~Q%7kw3PPH_jyRYW*!831S^77u*Mu%stDE(d|{OH-k^$PbZ-xNsK z=6+rJx-2{_bi2z#{pmjy)8;>)Ex>T$#nrv-o19jsok%dqv)+BLq{=bsX%o|*)}u>* zzJH{A|EPW0*SP=JOC&Wp(^`2FWGo8Le)iSqb$k5yvF`d6J$3U1{@NvW{T28vXtv2{ zQtg%M7WdkUr=kW&>o|&?84AAgFfbGg=U#etT3_Q<-3O-_CnJUnFH7X)-c8wZX3CZ` z&)?N)i}@H`xtfr7j^Wrg=I74JNi(@EpJ!)h_s`$|`I5E(Lx#z$28Q^wvbeOedCzBG z_^6z#-Ya|L(w&!`zn4bafA!No%~1DD{dp-1Ltvs}q2a_&pH7`xrDYcv$H-{MAngw>-Y; zl~h`ou4Ipb{rnv>JW5M_Gb1hMo^$>8Nm)t9<7MTyOW)swF25D^s65h<@!y}OM|=J( zy3lds`o4p!#eE+>S6}9@y6y0SEW>8d1}c?=?97nh=+_UWc^F)eE_F=2XeQS7e4XXQ z`s#1*R@#*QxiEbmV|8Dg%76c&uiqXo-hHi4?%I8Qi>BhWyQ02-RA#?Z`Jn36rC6c6 zXRGy=^W{WXU2i;E9o{fM|LUVmLwTX=IaVQ|z7n?6^K9?@eII}Nx&J3lj@2vfGhBYL z^wEnif6h#0tDfSvc;~)v+_loeKP&GS-r>02YW(&r&sN*}EqC_3SJGBqdCmFdvs&>@ z*>-XB>ch^RJ99F6>qmz;N5M!BGfCq$|L?ZvrF3qZ^ykgfXk(*d28TF5x3%q;M2{bz zo^Pv^-Ojtk!}GMt=e56HRIgm*hKGtn`YH z2CaSnrpz9*`}3C2Q}t6n@EiWVcU+yZA@2UYwR5lTJ-$BgPQBmq;$F!#<0Hp>7ER8` z&5b>AyIefarpw>s=AP$Y4(4jJ&iHrk^GE*mYgWxOt9!Ab@$Zj{O5@F+&&3w=adhq~ zJb(03h`XEDp>wxp82sMTtky4mW{+=BXz1Ig;#>^PYkV0NtXr4&{9$(?=gC!CU7OW< z1A+q=UEaB==+isfqVme8yWjt~+8iCOaiV_bychB^kugEn?%zphSh~JeX&5HJ0r*F%==J!i~DQf|S9~GTi3_ITZc(OD+LA~%|q;R|R>+Fx8K24b#x}%aU zX7A@K7jJIe+_R*2Cu8gC4J)1;vlR~&b!`tU+hjCJGIHmmBw5Q#XTz@-p1xJd;IQlT zjY#RUGmW~EuHM@E_)OXR|LZuoKKwN;Z`wFdZLXic@A+Mo|68ZWzgX%0eTm8uo)W3Z z|IgRi3y6u$G0NKz#lLjXBqPSUePVoRhcb3^E`Ix`Nx58h-wF-=>I(Lax80W>xUeF4 z`n!uSzj!9I^KLX16%<>wb}cVYb@{th#R(Z-UTXWDer6ePh-I4E;e>TZZG;?{WcwL< z-8P$Q&R(xkLxN&XGs#U)}ynManEIY$d)ztdGd)M!1 zneaTu`oysF~&f1KKH&g|~w5*M2qVB^dCyQ`Dyy=_(J^y%%> zr?>0Wzk0TK?b5UbYyJMIMn{AcJ^aPIinICPj;i0BKTjS$92*<^`A6&ZCdtQDCE*I@ zJsO@y$yqCO74>#~f1!Hn8FRyK{=2_sOZP~mc+RXff4_TznLNXjXX0UDp|7)7@80v} z?3r_{4ad*hMqb(WWO9Fh`SW8-xjxPPf3HsGnX}T(lnrYmLwx?+UcoOiX9j;Kr)OqZ z+OPU8H>XecpYdd_^!qclQ;Xcf-hJKO(zrkEq=|5?ZqPix=SCMUUU+ji`RlX8nwgR2 z?{=PSqtVsHm!%h_PY)0I77}=8XQ3g#dE)f)w@Qca1Rg(KUn*)qt!-u;c9=Izxn(e~8olWXE^Tti%5bguq>();_DU*DcR`X%&m&$4A@ z|LafB5Wn^G>g=$;3Z`?zV`J~0^#0C}@NrF{nb?uq+@jw%Bm6hdZn|@6|M$J0xMXav zcm@7D-C2J*(Im7Rw0y8cg$cF>|KyCXKlNRTx({Bv)M4^_is;kFLX%FP(wdR(d#lFb zPTMapU3vNUZ@H^y-k$XH+@(op(&EjJ>M?9cemrT}q)l_B^}`~DKC$G_jarsDtV z#5PgUsnz{)QF~KveZ6cgWVe*9&DFmCsrS398#SK)`FVdA>#MB0+}&P}?1Zc%84esi zeAhKBD$eiOLDOq9TI+w^wL2%SKAZXNGs%GP@bm?X9dFi5S;Am{e~#wjZEtKppVBV4 z)v|Djq078_UiljuZ{+3m^YcSf*WbBtrpi3WYVN)JcMiPl+&w#;Ve`%B&m3c{B{Fh- z%0908_vUI{3vcS@z}SA7oS%2@>MN+MTD7Zf`tro$kbnvGe@-w?ITLQYeowOmPutR& zda0iSeb4l6)xT3QBR)Vx^!LX8U*gN3y%cT>yu75vI?FAfVg6^Uq8Sz0*ZX!{xny(u zqcVGdnf%q4J)MVbYnqEcb2%&Dd;RsVkWs-8hvKx^{0z4YBvy$V3Cn#|G+)1=dyPoy z=Rl@SMw8rj&5EmAc&{$;%yy<(e9c)+lA!(FDvZ(IN-^`}?%&f{o+Y+ym;3p9Yw~J7 zFX3gG6l}~tch|KoiA7m%mk)i7?w_Q-=S*nk%bQQszj~YgTzM<@O{5W{@=+gCT-@W)!^G9QMtMT#omQ@o)mS43Z+3 zn9TIlH-Elttq(}?%(Q$NKWXArY0=LU=l5T~6e7+5+D)SLu&eOFldsI`ov%0g2K)Nv z`sFb`6<@UGMynCyY%X)%&fgv{>L0%Qe(3J{Ox?&y6K6WltTmBZJN=coegCsrB{M!v zTcdpY-X4g2#yLys+ud*9m6C0B_4i+B zSrQV>t?cMn_l(zG@nBoTkvZA+OP55?ep&xx#*sVIrnfI&nLB-oQ$*ud&5Xm_7@+3@ zY~A{B;*pa^GfyP1es*R7W7x4fCBJt4?^|;H{$XZ@{|`8fUVKSs?fp0RgXUDX#a{!% zdH=nU^$Gia@x{zp^IZSB-e1AXATOWqnJM|RTu)y9{j{y0#m`85KlInvGgGo#rTzTk z^gO$gj}M+dm$tld>g-{Ci$Ag84IO<`mRNkQVQkQq;hR;TdzIht_mwIQ*L~gmoc)s4 zYwAxg4-bgoVR)3Jlr#H6hKXU|!!>jEG#yep^+s9$>c$hs_aCk<-#hKno3%oMvd@32 zrF5*7WY|&3W+1^+EG+BDc&KsvmNko(wcTjC_GYb9y49iNz|%*+KHc8^Y}?|}pe(7b=%f6Ic@$tSk@$H@48_MR~H2ePey62o98&$4Pl;X6H zSH9k5#?uz$n#%R+{aFw1vPE^r7pz+0GyVK>^%Tz21{#KE6c|-5D00q__#!AIqA=HT zhPs4=RA^XOVs_?}r{cfPoK5e@-T(Bn`dhoA&XV${&!6Akomv?h`n9X~x8?O+d%no# zByX#@*7EoG`U&afOn+*s&GfZ(w0@Rf=X@qt`{!_%_bIV6TjIaIZ+tX~tvl;rPvJ7Q zTJd!^J?~nlzg_?|-b4zIa$z*xr7BqbpZM#Dp?--V6^9|9+wM z#wF!B-Mm}Fb>jcHHT3+QnlxWI%Nxhu-1Jt8V1MaYl=6I*HGj-)?>k52lfKQA z&b(j!MaxLd&d$(vX>|BB)y#L3|42_a`8Vs7fdtPduf`n`8)sf$m6Dz+BPjd)nWVvy zYdr@OCa4!S{}MhY)8f2B!QR|_@3LiO4|@vjUx>|}v4X?lmx8o#P5;7B?(d-w-|s7H zbyfXysdvxOlZH>k1=_0rTA5j|cyA<87h)*BvV$SAlj_tFMu(*xyZ#?zQem2*B*VC!t36YUI z8~fUJMSY+BJ@&?}nj_mzXYar0yoN6)`sJrf+gGi+b;e`%Y`=87seL`2+vP12ZnwDx z8p_u)AG+sOd!z8fop1GG^Z(yZD__6h(4r*W($Ky8_A)y!erKChpYi)$ty7xcy<=C4 z<15)}{!F>G)%&@l*|LR#hLXMKJ!hyNQj8CaEBtb%cWu4?tak=~&OEK|JEzZ6Y8s|L zGc9kCw5+tg>dlkK?rhkz=+I)j22OU{{r*>{|9*MD(C3ubr(<93B!BnEC*M=eyXEQ> zBqhqY$-P|ZmiJ}()J>rkSK3dl(o%F}ldnQ^I zK52Q==?fWK^0@fcnWF7r)6cxpmI@`T5w=B*DdY zkA)^(&$FB~ai($IlZKCr*;H!0V)yJ+t`w{h!>G(whs; zo}2JNLfd$6XEJ-akg#Rp!*BMt`vV<+YdlR^wrNw=(YC+*u{K*wuRimUOkJ?}-IpI% zm&K|_9*O$*Qdr>KQ}HbOLt$HYC;5Wr*4p*%)@bOf+up3}=&X}3pMStLyF9S&h;I1r z)hhKT^5hwG=Gxjl49<#j`_?vB{hH3qTdrZdUh*^_zW>&~uy?lK{atzcHci^Iw)Np& z_w`K9i~lw$N9O$q4GEdIn*Ux#a^i(AM)ARCbyV-%DS2J@i+^6_j!L#CPXzlz`Ty*> z>h5(bLgI+3%Khpur;aS(FD^)9`0#<(viRx09fI2yw?}MTk^1{&?EC7|MGqJ3Z*9DM z`MP7`MaRVVU$#mg-RkLiI%@mZ6%*GzYnu5usx*E=?t9tY6ZU+I4!ZSX?wj{J$`}f6 z-#mL|fqtd!l%~p~9}5dTBh?E7)7C4jnr3-<)xinoToKC_FJ9{AvQA%PSLONCU55?Q zo=y!7&E75}DkvCbb2@I_M@#+963-?w2)#JK(&c@M?UsICUXvZ;L$k%tgSXymVV4kG z)sy=u=J%z`^SO4Oxm0s(lJ6PSi|_68ZWPYh*>Y;L$E2qZ1K&^2zu|w9;lPE?dc*p- zy?~n4qdZa~^*z?mN@!`}xe_+eb?3YF}LW z>io5E!%DU2Z*vxXx_Z?#Jl}rdp$h>|->QCgF8j^k5D`&uxGVC~q`7u=Ua|WgKG;~i z{l44rr%%PNUb*t@nfNcZeRu0V{JHk``KeP{PprH>y@k#voH*s-CuiPqFyYPL`%|UA z&wqYB{_EuqvhKXI>Jr^xg{OlC-0{rB%->I^WyfRU;lA=2vlU@@?a; zPn(q4A`K*Z?z-#Ut^G3T%cAxDmU4UQZW}(*uan}GgbBZo z+%J6l%C>RUwLj%gd;?g1-cFBlJ9lvM^^@+7Vm|WDt~WCCR;-oc`N{28mbY(tK1cIG zkCpxI{>zJX=NL|_(0|hT`=asoOGX_p7W1$ve~SIHYwdBN$(Omgx>c=Bk4(?ZlZfbp z&U!ZJHp*|k&CcM(Rn%U-#;WYxN8f;!)$RMUYtM*0b1W#n?9Q{LKqA@wsEM5Qn)=Yv zv$^Tv?#spd{PbB^e`ss1KGA#Uo!6(=)6Zr|CA3B=w(r?9r^jvmyFf#E#skOhG`RB5 z^}D~J=Gi;t>B*0TzRT}FcJ%1c^XI+g{4Zy0krZx^{(kxSf;l@+U0HD6clq;~H!DAf zy_+(1qJW^-tvzXa3}*@?!r}s7{<`)5)!XIa`SzkhA{yH374lDiGrksQu=nEJu(Z}0 z%Q?QhU;h5f6U~eDKJCS4k;ykV*>9BLl4lUXw%Pa<)RL$r+z;E)q!`5E0@$2cdnjbU%+)}sD z`j+gL<`-JpJ7YyadZs6HZfUWw>@?v}sa~_pGY-nKz3KY3%E^z4KXbdfw(L4}GUDsy z%|@5A4}aAAyoEh#_Jp+i2YyYroU!^u!f9)By{fmue@QMariB7HX^vs24sPcX$4dUoEct z*4NLBa#*){%F>zZdRrO4S_gTvH5-AZ(XAP+dIL}Y5IXtrw&)@@l@HvU7iP7E>1k_w zdwM>6`gH1~td328xsSKU)+;w|*;Gqst2*puh=1cz)+nY1n2TyH-yHZp2z zZf-uDY{RslWohGSPP`W9|B8 zRo`YW`0%?w-q1eDae49H9lup)6!_-m7I_`}tG<5TuByqkvNyMeHO$Yy*OjXiyFS9; z=Cw;5sxM@_c^G7FZVlT!ohvdfGIZ|T$>Klt{w?fw&xuj$-q6~)?(YglEw9+st5!ST zKNh#+;hZ_MJWrjOWmEKNhvJl)z{1Rgj4L~{{UUExhCJ*)y(Z4)?5_EBmuGHTu$b+` zp}%M6EuOx@|5?x4r5Ep?PS=gvY;5*UxcH~}v&bI?+19l`@8v!h*J09ODEIf8XC1@0?pi-Yr~XaJ25li-mu; zzp<2+y7=PmvKQaw@4MA8Fj$%IUElUc0@@Iv6;{r&&vsmU&#=KL)83cJ9 zq@0{Tp@!#}+^S|pUpv?D?{l;Ehm(_#U|^%q-jLmYHz#dd(Q)B-+e}?l*0#U(TK~>73Hdw)1P|)Oc0?_5Q-toz^?wA}t^_RrKdA7JK{qYfdUB z+o`O(v0?wJT~Z7pPpsu{Z>~G_W8oS8keEFmC#+#*xPHBEx4G_Y^V&1p{-)nHSn~cx zi{#z%klR|JU)LA7PWI&~F+HWNHM22zRULbS{i~^bS(8BhX7Gxe58Rv`U0ttUzutb~ z+2d!=$_fg89J|QH$+aflOxw~l`KSHw8M$^09X}^nWWN;Szn{Ptd*#}-xw}fH^V~b< zxaMr=($uHF`(nR+;+|_&C-Ji+D?O8wvCgjiy~C+h-hOBQyneqo`up6>b<-zo2cO&Y zWy>t#14ZXp>O>vxynVb!FQOY#+r6FFuxr<^9!X=p*rfXV-IY~*e0-NK<;YD;pY}7W zZo$^h6&)9**)BgZy)NQOc2=i*xyAB5dz^}*`!?U$er4Xyn)7!e(i^6Jd>Y-qY18Ld zuU_$#n$DfKXVu!Z%2(^W{+)75&Z|?jm)D=x~QNCgN4@n6K1}R>uDksCmBQlm%#)JB?h-FnaDVu`_xFT? zcdLSPV-HV=vW~Zln|Jb?bIuuFRh1(_LB_J3&ad27{rLHFnXg1z?lOHny-%8!f1lZk zD=IdIb}zF!6FGnGY<`A4Ha3%w@ow5^Flmy=`P158Duj!!el)zDn)~(!Lj%)|wr#n2 zO=B$kB&Cl>J*7W+Jkn8pqm(CuJcZ)i7?v{)D>g!FRw~aSRvuoZj+{Ll@3`;tjfZBxZoTrp{&w9OrM^S^RSOw@=w|J|C3be-dL<|5 zk!vqFSklhUnmTEcknrUb)0--j?jNldXGpR3Ub-yRFnihMoy%_Ur|K~>_}@>Mrkrr- zb(7!qmzQ6!ubjN8`14-r-=&*O{(n2)vqt9A1aXFQ@x|xPot$r$cmq^au3u5}=g{kG zwbPppC2ia5(b26p&)6|A?%NN`KQE9bn_tym`n;e^VBk8?VwK(fWX00X;CN3=n4`IQVp;gcSrtds|`s>Ac zUQ>*!EjE>xmz6z>7Ji;{_418fge< z*OYzzc3%7xKmFn3$BUNz+1BIP7x+-kYFxI#ig_H|Z@OV7XIzEfTIvHX{=efCSYY8{`FpR%?dE%6M=VcPmHA~5o&qUBxl2duXHkFHyncCqLT zPj~3qXI%O-ulhcJ`qU~dwLaxkXh=xaPrJ{bYL~5FyDjnRovo+S|86R<-79(V>$dRH zQq$RI^)z(q{uwn%&M};NqV&}RlfoUXo%6O#aVad!e6!odlXK&uM?z6*jfGnz89s4$ zwXm@M{1IWhyL@4jag-3ljhwQGl%Bbv_Gya7^-p)D-%Z}!)Wj6~G{v6u%)Mi@A z`&*F?wEDQr@{P8;PG1h5q<(JqMfciF&viX3IzH@M ztF3zU=-m8@W5?9Hx;j>@)L3QlC8e#F_o!j%V>!L2_kT`4n)LGr>-|qPXRhYoecKbm z#9M2|@*&wS(r5bH6Sj*rZ?9x#s5z~1eJhK8!OGJt-rx<>9t+kAdU$#5s{e2I^lA&I z@R_q`Gx>ded}hoq`5B)6u9{s$L`-+4aiojOj&(wknI{XC-^B0l^!9%KQtsf4?eCYW z9GxmEI%AcKYs=LT5$Rd^$zEnt=9}jFgob`SeEf;j^UlTWR`YM@>0j^Ej0uQfVMt0^ zbmoxSXEv*>PepV2<`(TMcpDdRB3u6dgT?7F6?-mO8g|MgUR-IJ>QsItX=91WiJg=6 zy+lQ)W=h_)nx(QQO)XZpPs__8HgskrH^Y~zLRZ(+tg2}_>N77~H#{@n{Nnq#{hg1q zzV^;PsVR9>b?L8LwL8B^yqFNnUO0c7Wzp5TyXtMV@yB_i6~Se*L$=a`ckkxS_fOkA zW$IM-UMbV`b93g|p3F10v)lK~>ZIdi4IQm}+ur-Emy7+L8+UWHd-yIHhN`bGb`?K1 z5=>m!{8hFs!RyjbcDu&Z)Ti9(ej>k2!-nTw* z$YuTfMzOU43upg1V%WUmZcE9-?db&*zu%Tz#0E|C7g-J+I<#l6Wjp_|9!ckS$-+Xj zjMLBUDt)cyH|NGu?wdi^_GlVdiAFem2uIT=+>-mrFsV>`a^GzIAY^h18m)Gfk zH)&~e0p;<`_vmz?m1_Ez5KJS z?)~kSCA<1croH~Le}4U{$~nFcaqAY(D~mS_oOt@W?Tcd0!jJNObM~3NpZ+nO|F5Ws zj9*@t*`;?H%3J^6KYixt&0VuzPus&$J9*yBy#>#X-0R|cInz?Ab}Dblm05e3CS2S6 z`$OyB==$6HbFVH|KYz)1(^k`vlC2@iiyu#w=9Brl<#*7%i(ZQ!3Dx{*(%)V6#rQ&$ zUHvRq`@80b??rZaus~yETjrr`O4)x$UZ7HzQX#MaEyR;AK*7o`j^VzyA-* z`#bKRcPKvUq^@n;eB#CSO>?boM1IxQ)UB2I(DZKgP9t^wI~Ri9Z%%!CXo2{8dzIy< z%chvUUmhM8x9{_XozGr1U20B@|NO}%7g}{a;<&T>_|ClK0}YIAZF7oeojP@@`1!f5 zn;y-1ol~~BcK*H!;lFu57W{j5%<9`;-yc6BR!ZIs&$ro8_S(@|e6dMN^Yg{aGhaqd zKE^ATkf@??VE&9L+df~gd=@gltH@tc)AQ)OyRUeETa`|bvp>~n6*+0<)ZZ!(j{1H- zenxp0!-JyNiGO}SIr*3^de*DYpFg@DRefVoFpW=J+GtH)Wn|^iP4;DHXWCs`x4&}% z!~fSSwtHWDGs863$6a#eu0y5Xpd+aMZ!10aWMbMet3`(n1#aZrzwT~o>ZxVaTlcW8Ty^U5v5q&hR3axwEwj$J+OHH4Fj+)IUz&pG@u zTQZ&4R!=+-E^y=b(e&`Wd-wLstG_?i)!m(cXUE3q?Nxp5CMhRwYL}n0yq~vX+tVqt zzh90uNIs>G*hECWo&yaP<{I0WHIxScP*NCCVlo3cK0cN zvd_5VO;*N9x5dyT(ZI5A;(^WSa}KXkcJKSMXvTZ#-+%MYM(SMq(y9DSW^ae#eQ`ti zjO1rkW$Lvjm>G7JzqL7W?a$5b>uYVFWw$WveKKQWhn$+>v7l7=!}ss}ysG{>8rs_Z z5{8RPtuHrx%`vO~Ja2NRr?1Vz+YYb4Mazf?&6_#(_xWF{{qx&Y)RvaM)X7fH3<-<- z_&)Nw84v5>eP%X0UPyr&6$|zXfB5ociGYR0gtNK1rio|1iz?5a9>3$q_j-+V?hXt?R zzI`h#c0@|$!ak5e`;JNPbp`dfx3DwG$;r98KJkoPJnM>FjGO#K27MNTH;-F|PJvf- zPjFbRC?G2O^}#{rQ+Kx2Hf+Dj!zpzu31mo88xt>NiE-mI1qc22Q4tXt+1cjpURN4f zZgDX3{JwR+cPli+qdt_W*LjC?e(e?7{nR(Z13lWoOQR2qrmpzsda6aYED#f zfAoLwv(lsLqwl@Eb4(xoHeBZsV%qP?Zajpn z;K0p&VFn%n?PT)1i z-4cg%o*8FYnmsQZ=HR<|@Hqe3>o@x^Uy0;pTi^a9;Di0I$2~t1ZLO`XfB*Uwbk~2T zk!w`culAWuvdWFqycp7dN4+|+AQm(&D9*BK^#TLoq&){x6?3k)ZQ+;@-1u_3r$xs~ zQ(=qhZ*R`dHrM1|<}X|963sL;Galbt6B1ZZCf@C%Y#Bi8gS9@qRWQrds;EI{a+o*2g>yog9`%O>CAJFZOe_9$x3tmfM9>u8`V-g6ZiUWv*}iwJLlk3IvD?*03t(#j z6uz3W%xww}xBo1~uq#6#RI5zkDr;A<;!jbQKir9VTdSgi&YeGh{_^F|Z*Om(68JbP z$lFLD*O5uLo1v{%H{0fdlu853KJ&&0+y?I$Jj&TwnSTg4RXrDP>HP1sf4=9uvu&Gd zUMz8`<54^OcmKiK*km!SnTO9vD=50Boa%OnV5*XuEZ_bA;Kzv*1Y>V+n=Eu@A8)LQ zr)R0E1Lv1_{&gnpg~tFCaYFpjxXPWE3kv9M=#%H~@)?U}Qq6$i&; zX6>s}1E%=4th8ULr~B;Sj=u-OKmY%8sYd9MXF2oM#sm}5bG}MeE0>rG)$L^a<8)x1 z+UkdkE-kQ|@yEuVfBLdw>0_Cnnd_cDe}4Sv(aNf-w)XbRcazV!FFwE{l+nS^R?9o_ zAZU*A7JK8TB%_5_H_Mk?e%ey2ylvtPeS6WobqbqfL{$}(c$XO`+(?u7qCRtPb=k5y z6C>M%bFo4dtuZX$c_)avAE}Z$d_wlZajCRj2lbRS^wnCK4yjnZS>&0rPQOvZf5kec zn|4ne+AfBy(D>`Kzl*n{v|;)21Fz*TSu=1lz73nWa`(#T4{HT1*FM_W`6r`aZSNAd z1AKeWn`HDgd~9F1s_Dp@x^_>C6Z#xdhZEk|U6A2>%y($%EQ#qy+53)rp85Vp=T4O8 zAv?Re3zFVV35zv9{E<0g^T+3H|BoAcDvJXH1HHVwDl0AT?w>K^#so#@W4+Su(@nzD zc0K;^vhVP0Z|PT#O!jja+G_1UURk$>HAklT#ay`KkES?09Qj zxsKniPUth^fvYs^_Et?hx|7F@Hx}^Ax4SwrU~>{OHH&tn>KlV+oYWQqD4*GrB{`_w=--|oVa?~x6Q%% zlhQXfbX~nyaV6~H&Pz6%@@^lWrKiyNOn=>il(=Z8r>j<<$X6_USG4ZEoT=Q&l?CT2 z3$6yu-(=dh@n%7;+{}w5Kbp_DyV*DFIR4=nbJ6|lw_mi2WMyQ8goHpY!I(aMdU$B) z&Rx6O+S>Z0Oix|7@S%N1%Uq?#Fc*fy`@&2jGPHyonEo>|ItB(@P_LTod~*I3t@I*| z^r@Tg@vUSHSf`cMyJHQ*J^RP2R{mYW_qprTgulg4POw(`FPe8mCw*hWHQ!BN4b&Om zt!dltf1+x2Ncf9velu-{INk+mlQzle&hOGVTqJz5CEmGnT~#TsQSymnj%Nj*=w&y1 zK9M+XUB8L(qqO{{>=m!|b>eqVGoCqX=FG~kuR`nV>lr$_yKiq!Ki?&){pE|@{VP9O zPX)?wZmDFF__p<@q8s}}2BC~84HrX%TAnU-2s&#Rmv!#_%ZvJTFBUCG+q7Yi1mA7* zTk;Mk-HUcL#S0vHx?s<~PLq$bnh#8w7N@52Z1uXP#m(9LOwaeVADH|^ZTfsZvy0*k z0d1-*2O^IzYhC%_`<;|SE*GcFd3d#)CDl7ZjN|v(6DBF$o<&7P++sQbp`m-r-bN)R zCYF|dJ=n}%|MzP+FK@3{va(bYOOy=bo!i$RwXyIzF?l@rEjZy7m$TWVcQxM&^VB`7 zTZP=o?dC%6f*rh7o)?`wwUXOrk?m)?~qrWxJ<*pNXXTR zkL^0kjx$<{h7n>c+;S6lCo`PLR`s5LNMFxKuqe&)lb+^8#f72VK6|UaM(wFE6#e=1 z*)uu28Vk*p&;C9Y%g~x}ktY{J=jWbfhm!1t0yp0LI%mt$m>oMO={(SiKD6lg6aTQY z>#QqHXT0NCw)sTw1xrqu_`m&=P6wxmc+6O8HsQ0!#1^pXt5GHZ?zVwl!} zeubSc&XtHQ-|$4a!=rFoM!ePGrST_N&xPEY(HbnW>1tXmnK?Bt~;Y7{e!`wTI5g6<}*>T zu8)j4Ed@kWPw%U1PdF22bhKiU;al_9YMr-NXn+6lBk|>OX8C8uC$(j-1WwxIBp~qU zN${Z)pB2AE#-0`Yu-W+H`E_c)Wm#J+_4}5z98|nrwQlyxug|NWO37VVvSdl_?QKt= zJYhI+W~TAmH*fY$_r1X{lwrzJuxo#>{qmvwLvq3ul~dV^!jUy4XV@78w$0XC-dsDS_4W1j{k^K;S8v?7ac17# z4^O*wkIHXh=Bx>4s%Si2ezbQkcyNrPab1)_1@leKij;u2+nl(1uXzZ47ELXb<8X_M z_egkgVd3v@ZaW-Ah zuJ*fv8YdLK>vnrgUVNnd+Js2kGE>X#i8j4^esVL+t1oPCe%p63+$tw2DJdmIL`H^( zK~7HYzQi|1hZnDBtT$nut=K5<4mNde@{MqPF&!6BN9@mZ7B1rxUr%q&pZs`>_xwqB zTT)pSZZ#JFxSr1ceLDz1BVN6dC7nKA6ZiFL0p( zsN1+el2e7{kXUKakH^j&mZ?fQjFp}r9&x>S&u?Crd#vu(Lap4~TyAdeQ-%EfzvS%l z+5;}6ssuPQr2qb)&jd2y0lT2asV4T>C;68>Q+x2>frIUJX<6B~j~^Q+9B6p+0%yJ(5I)GQwF5-eiX#?mY)yC31_Kv8qkq z$jC@XXU*Ony>&{BZk!1>u4~Ax0efAc@tWrX|M_-lX=zj5_RHJn&GlPw)nma-keV|N zfwCZXF>r1XVEXbqhmny{RdxQnE7^+27D{?Na2GhR;k|pnaXZLa0uT19FIc#6;p)}i zUyI-0+xvf?^BYFaEhV73^oyX=Z^%ILgVmfp5{6A#uOB{~ICbjRt5q8mg)*c;4wio0 z5eMz`+On8r2{DL>ipK7*+bhSZyt`S+_ZC`vJm?0gaorIh0jiW5Sfb<` zPfk`3cC)YfVPI=J_oYK+rR8Vg&mIeMg%504pWClr)eh;$FIc^L^~#l=jXUc9+xae8 zvxdj~&qWi1%W|M@xSX40EjS#vlrt@M?_VacWB2amOO^!ubZ+OnyQ@@NQ*-6s%Z~qV z^MZQhX9E^521Q$AnU6zo@Z~F4rZ9bYaFCh7qWoP%E?_R#Fe0fQgpP&Dgv4O#dqut{1XO}GQnKJ$MM!mV#wOiY!-6{Cj z!@2xhe!|9x7&k9(&x)P0ukUR&4zJuh-*|c7w<7Ch7sGenyQRh*>-YXee%^&861o@e zpLuZX=iRTj%#s5BPyT(4`M{l*kI%{fjCOEn$P3LscI?uP({E=yzTtT0R%my9+{Oc( z3@aAi-!7{n!Df6PJjOg=c202%t+<4pSin^9CcZEY`xnNh6SruRo(iLxM^;Rl#ua- z!0iSXvsZ{QtU1f0t)<0Va>eSDM`35~ox+@xvt(DBy{rCiSNZx@>!~iT7^YJD_FK_+ zs$Ya}pL1M@=XV85YZ5398C5R4;=Hi_OHrkey!`u@FHeSc%m8pp1P;6%KH8D zkG!7uzI>QzpWHNIL4q?=*|ooUl{Wt-%*ngGUw*oe&z;}b^B={vy1%!m_?GK*$wAs! zk8z!UX{c&u=E;WV3uB({sW&s+dvdqZ6mXIREloefdE)fx?#@n8aq;V0FJHZS_0pxF zn3z3zqVHAJzppLd{o5mWZU5Anb#;ypQ^Wgj&)awUpLyyJ1*6k1o|e6Fjkt2^YjsO2 zOS*OM=8Z-|VnS7a)!uzrnceBRGDP3LcZOo2TbZnmBEiWob#iYU`Xke}3O%6km6D>a;04%Bt8IR(yMBT7RrX zHM?w2o$&JYYt3$F_9UHuw^ROeezjd$*>3-{w*zY}ePz}yzxw^UeR<&g>HqIa{rzSu z`h@et=iPJf?@jz2c=g`i9$u6B*xj$s9Q3u_bvih1&c2e*+m`DyXq3F(lhha-FM9OJ zlrPQdbDWivUTxa6-HfwN^ySmvIR|?sWhXxpw=o0s(lzPb6Dy6bd$+S7X_-*?tM7o2_eT0p4X zhT5rjXY6E9(9tm||79WVom^M9&)t7sy@%i85ZgMbNmHEGeR{ce#fC5S!68$spVge% zU3mWat+VVtE({793I!jIOck!a{xWN2+rD+j4@#Hs_2t>3)+=4}?NqLp~R?IxMs1ojuozRgV}F&7K?fZ?3gzaxw&! zl?*}|Us?EMtwQq8ursu_ww~Q|#k%;_j5l}swXePXRwJ?@=#FyS0se zz0L2gPZNsYc;My7%Mu$ce{e9jN&l0vKHBMe$e&CX+wcgHM{y#Hc3dRYNc!Fg8moEy4+vi$g2{`c$a-AA4rUH#wkc}@DO9}A;i)$TC)@NuJ^%JSWJ zK1JD73)$D(1_p~Hv+}}J&BMo! zA3uLyT}8v<%E3zk;dkv{h;E*yt*@n&lbiTMS9?#2%Z@3J)FD-~LZg}IgCm{74F}8x`f%b#^>e!!9YS)NmUhnD*LUlx^!Llg#W!-k?RF_D?3vZI zXyLkJODtCx+dpG=NGbWV@~<@8%~`we)|yn_nNWLNukxoIbHj@rEP;Z8&k~ugTgQdx z7wF0UseJhBkdjflr1*cw7@w3Q3a`EO-n|zJKBcWyWcMsk)_1nWn`y@%b6?}3@O#lNr8`2J+p2V=u4LV{vdQJXb>-mp*k zx=AuAN$Kd(rBiP0T9foxc&@XU?3v`}ca$t)3M-TIRBQzR*!n*pT{m z5%cl&JF0(PxAWZ5F(vR`MpRK#UT~kYLyLrtz_WPgIBcG>F|y?^`h8){M|#-^xcX9PjQr_4KJ|TH5on zHyXa#i;Z8cI<;%os^+&A=}wV4(`KyjSl(y!a^}q`JhRX8aaG>+kDuI9>vk(V|Bk+{ zwz26Y&N|V|CF>uSU7sCxHK48QIm3c2TXgjIPv~NudFb0UHN}M!Q&XS*`BQUnz0&H{ ztA5Xxtf;VEn(``j>9R#za^Gu4dA@i(qyH!Cm*<-&To2jjUS6r`S@?cdN0(Nmq+Dn~ z#F~5Img?=BSNq4hYMT9HHukIfW0LYD;hgkYhJc``vTvu}_O5>3>UzYk`s1&O?>D!V zu}qsbP1dqF_vfC>xP9k$gf*_b$3MTst~4Rpo>@0-ZPNL?y`rLJvR3}S!gib$h9|DQ zxw3TjmNU0>+lz`mYiX?xx@076-^|2tSZ2+-`MPI4GTvM~bAIE&#Lww|Q>F;K zUwlPbMQ7ELHGi&G2J6S{y&b)gT{gk+>CR7EJm-8m|5iX;*wplrn2^Yx_eCvv8Tq+R z?X_Pwttt-ZF^~+anbq*5$@%qiWv!r~aBWPWRpO;*pEf`@2|cd(I!fX18g~y;FAmEhlvC2LtDpET&_} zjuk&YclF}Mhex~9ZazGIS8(>(*2Gv?t5U6S5?WK>SrvXvw3^^?qz;^>-+uC`Sa^9_|CJFbNnhK z^J`f$`^rM)H;0sxy}eH#iaYM%?bRcjC@nkvz?|g8EjM$_?C(FHD4UyIJM-T&w$RY4 z-C?=&_ZM-0-TeK9tL@r-y)kV}4Eag-BBLcP{VLfS^4d@7+M1cmo{EdyjF7rG>DR5S zNkU11hVNfYS}~(s>y+&b@gI@1uV=9_1PUfrXHVYJ^JextyVKX-@^RK0*C;&il8@pv zE`IPs^!C4FTBmGhwwBzxd3JJtfBuyTj3J!eM>lOX+E@2o@a?}pQ==a}vMnkZbkTsW|6V)0VZ(+ETep6FzyJTf{QGw9?#JWyR+&CJ z-_9?;Bkyar%Z%fzzO0c66nv=oP3)GG)PC@JlQK_S@ina+Sz8g zQbs8qJwAWSKkVAItM1Q_jgp#ss^{1I+;#jId6DU)bqrmiBwwAG4>+ z7fz44(|=uNS)y0ookE=-p-;M4{cc~^UAbaSg4}Y3hBI&OOnv=b;F9Z z{Uvw$cC~f+Hsw#vj5noxV`X4Gf0)TW_u!Vz@5IFfPj78|{q(m_h|7s990L2#adlNyPEO8~r>+%?>?}OJVcSx{1#vRrp`lNYZUC>L zsF>(J^PBU61!|fa8XcXTAD@(h7EnC&6l$GtKwR*K_q^_mV8XYXN)EW}wFZTp6K?W{vaWYm`{=8_asivl; zprBx?`LNn5E;O|E*{8S0)A!%s^y&K-zTa7#EFV1Fy(L(;FhX=ii7|$TUVV9ad3k6+ zPGs6bQ;!F?K?5SWe>RvxTauvFY*CzyJint(IxL(Caxh3_0)x;GGbV{|wbJ%E(3K4y z3yumKT)O9eTlVj4{M`ArS7W2W#od*O{`s!0ZYeLc6dez8E@qdz|M~e| zZT36o-p1`O((C8VS4`Tz zwom6*E7XDp0Rb_gEhbm(3;vus)#_I}rTWOx`XBFigoK9Mf8p(_zFs1}dipsHZT84( zt6S4>!3NT{aA3Oc0$MY-Y15~Nj}JJSrG2}$tmKCIm4ihq_CCET{k?vskyqTh;Fy|6 z8#k{L49sUd@bk#GL0-QuUe;q!P}SU5aZbfpd2ZLKQ^)<|Wvm|u#RPeohX)6mTnU|Tmai-O zIWXa)86yJ&#|uvv$BZAGAr_282 zKHisG?f*J!RiE6io7uf=R<8m#a;|(C=`;7t9|QK)qTF+A_A*afceC*1)!I}3&gs^@ zOIfT=c$}MK^lQ%NxtqiD%v0EcMHxPP{#a>O^~zvD+1%@i4KHioGt~Vn6K8#xBY*uq z?;rR;OBsiUx2MF*%r!rMGcf#M+Y@1O`=W!puF4nQrwdzlyqIMAf6xC3TnrO0ew0mF zqShzV3)KJYE{`TBbcck)PQd^KF3@cG^3E8ljMzn?tU zKkxnR<>4iFs=byq_k_H7Es^!kK;@0E>(O-MR@oGl;-b$PxiONAjE#|(kI7in9ld*6 zo?+4Bd&|?G{Mnq?`Fd?TgMy;ox7%&C7Ym=Lz3oh9GtFaL`}5-D=bzJk0)hj3x_kp( zsDYXgALl0vyIEggdaKf$7;i0^Q6dZ}JZ>p7N_#(kG-=X?j)=OteNBfJeK=uJogHlT z=J$j-ZbxfGW*zr;R_m4Zc{-nE-;zq#V@J}{p08T16S8b7Z@>6mTmNm}*;XaxsQB<+-NBEXo<1k<^s~NNb?S^z8nl0qf2`^K=?^QK4kdlv zx8&sUKTKNBm>bm2F1RWtCi>gH+VsLP9WArO1-&o3H>^1GCT7>RzbOW9vh9<5=Yhw) zCphd@PRP$y(o}p`_f_F-rDx{Nd&%ObPn~)xWOX=cqmLu+rwjINt}{2hVq|#5>d*OC_2+lR?f>1eu2%3{+Oa8tu}@MjzlwgIs^@&!mtny% zv#?ZG(ajqQjb}`n*UuEDKezD77t`wO$&ishr<>gPG{(f_s?ef04rIPlq zHb1?x;zrAa3+CC=&It$$FRWh3v%%=9zmL~xeT_{UO%|>Gvh}Kp>XAiB_b#f2EGsnL z(b6368Ws|AW}aDCP|Tfv!|5kguWhq0+=e^~{&sr9-mRWdRo9s3D?F3AclmkZw`fzaMXC4BHj8>T_7m!V1Y3tMq5R{c%8j!|FYE_Qcm!JrJHT zXV%5&&SL)0bARtsW{>{AH|^ik*K!O7PnRz&nwwJd$7w;Av8ega<%-E|H|Onn5EdDl znzm;4lu74Id7{71`;~t9l5q9)yL+C0HdFKrak+8o%zXZV>?2!N9Xq?= z=!>5A>z6x|*(&X-yrvr%zYagQ)c)`M)^)e5X8g-BmVCFgO6WJDU3F@kr2Xx+%9@%_ z>=QymL*qD7uB)7!b)?(o)E0RrR#Q($M@Ju@9`*H)YaTpE|8Dx>3Rmj{hsiA8Hq92v zbZguk{%%h3zPmkK(Nk6}Q#Gm8~3gpgVvuY?Zlt%WYb0jt{P5At+1|DRW1iyy5C{W>i< zFePdQ^Meb+VY_(-3k8r2i?RkaWqxxk`U^4&+LU?F$o+obeVkQ}tH4;~UaJ+YRbW65l?yaJE94 zDPYYPZh;zpx7-EZx`P{j5X=2RYdblGxgb>`_>}JjTZIp7SiiMj;Z{F1(Sh{O<4Cx1 zTv$y6vM&>?*&$oKAu&E%-f@dO$aT;zqm7_IhRdooYt~GRT9b8{rQpTw4-Xb)LtK0z zkJCir+tlgP)6>(pN9(=4`LU^~>DwE-U-xoYe(_|Og|6V>{A0niD{7Vfy5cvIoF>MM zVzbXa3yKU4ojPq=SZwUs(7yR=r!*{BkT6{@w!(1ViWM5_>h0p5Q!?9Ge*E~ctK{XO z^XKpL)w)jy2U~8lgKc8`?E;>?kV!bOvl)aEKr5?!q(Jw?fV>6@c8A%j4Tg__wJL;7#J8BJYD@<);T3K0RY8ZF$Dks literal 0 HcmV?d00001 diff --git a/tensorflow/lite/g3doc/performance/implementing_delegate.md b/tensorflow/lite/g3doc/performance/implementing_delegate.md new file mode 100644 index 00000000000..85904cad091 --- /dev/null +++ b/tensorflow/lite/g3doc/performance/implementing_delegate.md @@ -0,0 +1,171 @@ +# Implementing a Delegate + +Note: The API used below is experimental and is subject to change. + +Follow the steps below to add a delegate: + +1. Define a kernel node that is responsible for evaluating the delegate + subgraph. +1. Create an instance of + [TfLiteDelegate](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/c/common.h#L611), + which is responsible for registering the kernel node and claiming the nodes + that the delegate can execute. + +To see it in code, define a delegate `MyDelegate` to execute Conv2D and Mean ops +faster. + +```c++ +#include "tensorflow/lite/util.h" +#include "tensorflow/lite/builtin_ops.h" +#include "tensorflow/lite/context_util.h" + +// This is where the execution of the operations or whole graph happens. +// The class below has an empty implementation just as a guideline +// on the structure. +class MyDelegate { + public: + // Returns true if MyDelegate can handle this type of op. + static bool SupportedOp(const TfLiteRegistration* registration) { + switch (registration->builtin_code) { + case kTfLiteBuiltinConv2d: + case kTfLiteBuiltinMean: + return true; + default: + return false; + } + } + + // Any initialization code needed + bool Init() {} + // Any preparation work needed (e.g. allocate buffers) + bool Prepare(TfLiteContext* context, TfLiteNode* node) {} + // Actual running of the delegate subgraph. + bool Invoke(TfLiteContext* context, TfLiteNode* node) {} + // ... Add any other methods needed. +}; + +// Create the TfLiteRegistration for the Kernel node which will replace +// the subgraph in the main TfLite graph. +TfLiteRegistration GetMyDelegateNodeRegistration() { + // This is the registration for the Delegate Node that gets added to + // the TFLite graph instead of the subgraph it replaces. + // It is treated as an OP node. But in this case + // Init initializes the delegate. + // Invoke runs the delegate graph. + // Prepare prepares the delegate. + // Free performs any memory cleanup needed by the delegate. + TfLiteRegistration kernel_registration; + kernel_registration.builtin_code = kTfLiteBuiltinDelegate; + kernel_registration.custom_name = "MyDelegate"; + kernel_registration.free = [](TfLiteContext* context, void* buffer) -> void { + delete reinterpret_cast(buffer); + }; + kernel_registration.init = [](TfLiteContext* context, const char* buffer, + size_t) -> void* { + // In the node init phase, initialize MyDelegate instance + const TfLiteDelegateParams* delegate_params = + reinterpret_cast(buffer); + MyDelegate* my_delegate = new MyDelegate; + if (!my_delegate->Init(context, params)) { + return nullptr; + } + return my_delegate; + }; + kernel_registration.invoke = [](TfLiteContext* context, + TfLiteNode* node) -> TfLiteStatus { + MyDelegate* kernel = reinterpret_cast(node->user_data); + return kernel->Invoke(context, node); + }; + kernel_registration.prepare = [](TfLiteContext* context, + TfLiteNode* node) -> TfLiteStatus { + MyDelegate* kernel = reinterpret_cast(node->user_data); + return kernel->Prepare(context, node); + }; + + return kernel_registration; +} + +// TfLiteDelegate methods + +TfLiteStatus DelegatePrepare(TfLiteContext* context, TfLiteDelegate* delegate) { + // Claim all nodes that can be evaluated by the delegate and ask the + // framework to update the graph with delegate kernel instead. + std::vector supported_nodes; + TfLiteIntArray* plan; + TF_LITE_ENSURE_STATUS(context->GetExecutionPlan(context, &plan)); + TfLiteNode* node; + TfLiteRegistration* registration; + for (int node_index : TfLiteIntArrayView(plan)) { + TF_LITE_ENSURE_STATUS(context->GetNodeAndRegistration( + context, node_index, &node, ®istration)); + if (MyDelegate::SupportedOp(registration)) { + supported_nodes.push_back(node_index); + } + } + TfLiteRegistration my_delegate_kernel_registration = + GetMyDelegateNodeRegistration(); + + // This call split the graphs into subgraphs, for subgraphs that can be + // handled by the delegate, it will replace it with a + // 'my_delegate_kernel_registration' + TfLiteIntArray* supported_nodes_int_array = + ::tflite::ConvertVectorToTfLiteIntArray(supported_nodes); + auto status = context->ReplaceNodeSubsetsWithDelegateKernels( + context, my_delegate_kernel_registration, + supported_nodes_int_array, delegate); + TfLiteIntArrayFree(supported_nodes_int_array); + return status +} + +void FreeBufferHandle(TfLiteContext* context, TfLiteDelegate* delegate, + TfLiteBufferHandle* handle) { + // Do any cleanups. +} + +TfLiteStatus CopyToBufferHandle(TfLiteContext* context, + TfLiteDelegate* delegate, + TfLiteBufferHandle buffer_handle, + TfLiteTensor* tensor) { + // Copies data from tensor to delegate buffer if needed. + return kTfLiteOk; +} + +TfLiteStatus CopyFromBufferHandle(TfLiteContext* context, + TfLiteDelegate* delegate, + TfLiteBufferHandle buffer_handle, + TfLiteTensor* tensor) { + // Copies the data from delegate buffer into the tensor raw memory. + return kTfLiteOk; +} + +// Caller takes ownership of the returned pointer. +TfLiteDelegate* CreateMyDelegate() { + TfLiteDelegate* delegate = new TfLiteDelegate; + + delegate->data_ = nullptr; + delegate->flags = kTfLiteDelegateFlagsNone; + delegate->Prepare = &DelegatePrepare; + // This cannot be null. + delegate->CopyFromBufferHandle = &CopyFromBufferHandle; + // This can be null. + delegate->CopyToBufferHandle = &CopyToBufferHandle; + // This can be null. + delegate->FreeBufferHandle = &FreeBufferHandle; + + return delegate; +} + + +// To add the delegate you need to call + +auto* my_delegate = CreateMyDelegate(); +if (interpreter->ModifyGraphWithDelegate(my_delegate) != + kTfLiteOk) { + // Handle error +} else { + interpreter->Invoke(); +} +... +// Don't forget to delete your delegate +delete my_delegate; +```